diff --git a/.agentenv/config.example.toml b/.agentenv/config.example.toml index ffdce8220..7ca405d01 100644 --- a/.agentenv/config.example.toml +++ b/.agentenv/config.example.toml @@ -60,6 +60,10 @@ agent_default = "local" # agents too # default_a2a_agent_id = "my-agent" # what a deploy_agent step that names no agent deploys; by default the # # agents bundle's Claude Code, @local/agentenv-framework/agents/claude-code +# [trajectories] +# formats = ["atif/1"] # the formats beside native a run prescribes to each agent whose trajectory it reads; +# # none by default, so agents build native only. Per run: user_overrides.trajectory_formats + # [envs] # default_gateway_env_id = "default" # the gateway every gateway deploy runs # default_service_db_env_id = "default-db" # the service-db local Postgres state runs from diff --git a/AGENTS.md b/AGENTS.md index b981213f9..48174a839 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,6 +90,10 @@ or one of them only where it is used. it records it: `cost_usd` with its `cost_status` (`priced`, `reported`, `unpriced`) and `cost_source` (`litellm@`, `harness` or `provider`), the `uncached_input_tokens` and `unit_prices` it was charged at, any `cost_assumed` that makes it an upper bound, and the agent's own figure, always, in `reported_cost_usd`. +- Core's own model calls, the judges', go through `config/model_call.py` (`resolve_route`, `complete`), the only + module in core that imports LiteLLM (`tst/unit/config/test_one_model_call_path.py`). Each call returns the model it + called, the host it reached and its cost: what the endpoint billed, a LiteLLM proxy's `x-litellm-response-cost`, as + `reported`/`provider`, else the map's price, never LiteLLM's own estimate. - A host-mode `install_agent` (no `container_name`) on a `local` sandbox runs the agent's `install_commands_host` on this machine, with your privileges. Every command a local sandbox runs carries `AGENTENV_SANDBOX=`, and teardown stops the processes that carry it, then removes @@ -114,6 +118,11 @@ or one of them only where it is used. with websites adds. An id set in code, by `configure(...)` or the config attribute (as `agent-env env deploy --gateway` does), beats the file. Within a run, its `user_overrides` (`gateway_env_id`, `service_db_env_id`, `website_browser_env_id`) beat both. A multi env's `browser_envs` names its own website browser instead, `[]` none. +- `[trajectories] formats`: the trajectory formats beside native, such as `atif/1`, that a run prescribes to each + agent whose trajectory it reads (prompt_agent and always_on call the trajectory extension's `prescribe` on the agent + they prompt before its conversation starts); a run's `user_overrides.trajectory_formats` beats it. None by default, + so an agent builds native only, and an `atif/1` read of a conversation not prescribed it is refused + (`format_not_prescribed`). - Secrets never live in the file: use `secret:KEY` or `env:NAME` references, resolved through `[stores.secret]`. - agent-env has no stage concept: "dev" versus "prod" is which file `AGENT_ENV_CONFIG` names. A @@ -131,7 +140,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register composes only the computer envs its tasks deploy (`materialize._composed_envs`: each env a task's steps name, and each env those hold but a computer env's own apps and driver, which run inside it) and those it pulls in from an installed bundle, whose one write every bundle shares, every one for a plan with no tasks, any other composing at its first deploy; one that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` (its driver draws each input over the frames in its caller's colour, and a pointer per role where the frames lack one, unless `annotation_settings` says `is_enabled: false`; a driver whose card lists no `annotate` records it plain, with a warning, and the run's recording entry says `annotated`) and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent, a program that isn't there exiting 127; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; on a multi env, `load_artifact` with `env_id` puts an environment artifact named after one of its computers, or after an app of one, into that computer, the computer's own as files at their paths under the session user's home; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. Its `headline_verifiers`, step ids saved only when set, name the grades that are its reward: one is the reward, several a multi-reward over them, none a multi-reward over every grade; it refuses one that names no step of the task, or one named twice. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. `task/lint.py` is `agent-env task lint` (a bundle folder, a task file or a stored task): an error is what a task gets wrong before it runs (a field its step doesn't read, a step's own `config_problems`, a headline that grades nothing, an aggregate source no verifier writes) and refuses `agent-env run`, `bundle register`, `task create`, `task validate` and `eval run`, and the bundle plans `openenv`'s `build_app` and the trainers make (`run`, `register`, `build_app`, `prepare_bundle` and the trainers take `--skip-lint`/`skip_lint`, which lifts lint's own rules but not a step's `config_problems`, which refused a bundle before lint; `eval run`'s lifts every error, since it checked none before); a warning is what may be wrong (a step's `config_warnings`, a store entity read with no version, no verifier, several and no headline) and is only printed, each kind once by a run (`task.lint.said`), an eval run's unpinned references as one count (`unpinned_summary`), every one by `task lint`. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` (its driver draws each input over the frames in its caller's colour, and a pointer per role where the frames lack one, unless `annotation_settings` says `is_enabled: false`; a driver whose card lists no `annotate` records it plain, with a warning, and the run's recording entry says `annotated`) and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `computer_exec` runs one command there through `run` (a program and its arguments, or a string through `sh -c`; in a multi env, the computer it names), recorded under `computer_exec` in the run's metadata, a non-zero exit or a timeout failing it unless `check` is off, or with `background` starts it and leaves it running, as an app launched for the agent, a program that isn't there exiting 127; with `scope: "system"` it runs as root instead, through the computer's sandbox (`RootChannel`, `docker exec -u 0`), in the session user's folder and `HOME`, its output capped in the computer, never with `background` and not on a computer booted from a disk; on a Windows computer, which has no `sh`, a string or `background` command is refused; on a multi env, `load_artifact` with `env_id` puts an environment artifact named after one of its computers, or after an app of one, into that computer, the computer's own as files at their paths under the session user's home; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. A run whose teardown leaves sandboxes up, stopped by a second signal or failing, keeps on its instance the context it ended with (`record_teardown`), and `agent-env run` and `task run --keep` name `agent-env task teardown --instance `, which tears down what that context still records up, a running one only with `--force`. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 688a5ad19..f7b526404 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -612,6 +612,7 @@ from agentenv_protocol.computer import ( class MyDriver(ComputerDriverEnvironment): buttons = ("left", "right") # features, on the card max_click_count = 2 + screenshot_delay_ms = 300 # how long after an input its screenshot is taken @display async def size(self) -> Display: ... # width, height, scale @@ -668,7 +669,11 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope `computer_scroll`, `computer_type`, `computer_key` (W3C key names) and `computer_wait`, plus the group tools. Every result is structured, with a one-line summary as its text block; input results carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame - after acting. Click and scroll leave out `x` and `y` to act at the pointer on a computer with + taken `screenshot_delay_ms` after the input, so a smooth scroll or the page a key opens, which renders a couple + of hundred milliseconds after the input returns, shows: the driver class's `screenshot_delay_ms` (500 unless it + sets its own), or the deploy's setting of that name, 0 capturing at once. A driver's own tools don't wait: they keep + their own timing. Click and scroll leave out `x` and `y` + to act at the pointer on a computer with `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. With the page group, `computer_navigate` opens a URL in the active tab, `computer_history` goes `back`, `forward` or `reload`s, `computer_tabs` lists, opens, switches and closes tabs by the browser's own ids and @@ -719,8 +724,9 @@ other exception from an input operation is reported `delivery_failed` with `appl **Settings** reach the driver as one JSON value in `AGENTENV_COMPUTER_SETTINGS` (`ComputerSettings`): `default_space`, `default_norm_scale`, `queue_wait_s`, -`tap_hold_ms`, `type_delay_ms`, `max_hold_ms`, `layout`, `drag_duration_ms`, `exec_output_bytes` -and `exec_timeout_s`. A driver reads them as `self.settings`, for example its tap hold in `@press`. +`tap_hold_ms`, `type_delay_ms`, `max_hold_ms`, `layout`, `drag_duration_ms`, `exec_output_bytes`, +`exec_timeout_s` and `screenshot_delay_ms`, which replaces the driver's own. A driver reads them as `self.settings`, +for example its tap hold in `@press`. Every caller is `default` until agents carry an identity of their own; `caller_identity(context)` names the caller whose latest frame and others' input a call uses. @@ -1623,9 +1629,27 @@ responses, the format it serves. enable(TRAJECTORY_V1, live=True, native="my-cli-events/1", atif=MyCliAtif) ``` -The card then lists `atif/1` in `formats` too; `default` stays `native`. The -SDK keeps one ATIF document per A2A context, run through one adapter instance, -and feeds it every event the agent appends, as it appends it. A native +The card then lists `atif/1` in `formats` too; `default` stays `native`. + +The agent builds `atif/1` only when prescribed: a run that reads its copies +says so first with the trajectory extension's `prescribe` +(`POST /ext/trajectory/prescribe`, `{formats: ["atif/1"]}`, each a format the +card offers; it answers every format the agent now builds, native first). + +- A prescription holds for each A2A context started after it. A context keeps the + formats it started with, so no copy misses its first turns, and a later call + sets them for the contexts after it. +- It's one setting for the whole agent. An agent that serves several runs builds, + for each new context, what the last call prescribed, whichever run made it. +- With no call the agent builds native only: no ATIF document, no media spool, + and no turn count of its own. A snapshot restored into it still records the + count the snapshot carries, so a context prescribed `atif/1` later numbers its + turns on from there. +- An `atif/1` read of a context that wasn't prescribed it is refused with `409 + format_not_prescribed`. A read that names no format always gets native. + +For a context prescribed `atif/1`, the SDK keeps one ATIF document, run through +one adapter instance, and feeds it every event the agent appends, as it appends it. A native trajectory returned at task end runs through the adapter then. The agent's own code feeds the adapter's aux files, records that aren't in the native log such as a hook's notes, with `request.trajectory.aux(name, record)`, where they diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index e6bf156b1..5362ddc03 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "agentenv-framework-protocol" -version = "0.1.618" +version = "0.1.624" description = "Open data-plane protocol and server SDK for agent environments." readme = "README.md" license = "Apache-2.0" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 99910083b..abefb4f10 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -79,6 +79,8 @@ TrajectoryEventsResponse, TrajectoryMediaRequest, TrajectoryMediaResponse, + TrajectoryPrescribeRequest, + TrajectoryPrescribeResponse, TrajectoryObjectsResponse, AtifEventsResponse, AtifObjectsResponse, @@ -260,6 +262,8 @@ "TrajectoryLog", "TrajectoryMediaRequest", "TrajectoryMediaResponse", + "TrajectoryPrescribeRequest", + "TrajectoryPrescribeResponse", "TrajectoryObjectsResponse", "TrajectoryState", "TrajectoryUploadedObjects", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py index 53beecc59..91752aa04 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/_trajectory_log.py @@ -6,8 +6,9 @@ import logging from collections import OrderedDict from dataclasses import dataclass -from typing import Any, Callable, Sequence +from typing import Any, Callable, Iterable, Sequence +from ..atif import FORMAT as ATIF_FORMAT from ..atif import Adapter from ._atif_copy import AtifCopy, AtifPage, TurnStart from ._media_spool import MediaSpool @@ -21,6 +22,15 @@ MAX_TRAJECTORY_READ_BYTES = 4 * 1024 * 1024 +class NotPrescribed(LookupError): + """A read of a format the conversation it names was not prescribed when it started.""" + + def __init__(self, context_id: str, format: str) -> None: + super().__init__(f"context {context_id!r} was not prescribed {format}, so it has no {format} copy") + self.context_id = context_id + self.format = format + + @dataclass(slots=True) class _TaskRecord: context_id: str @@ -104,6 +114,16 @@ def __init__( self._tasks: dict[str, _TaskRecord] = {} self._contexts: OrderedDict[str, list[str]] = OrderedDict() self._copies: dict[str, AtifCopy] = {} + #: The formats beside native a conversation started from now on is built in: none until prescribed. + self._prescribed: frozenset[str] = frozenset() + #: Each started conversation's formats beside native, fixed as its first task starts. + self._built: dict[str, frozenset[str]] = {} + + def prescribe(self, formats: Iterable[str]) -> frozenset[str]: + """Build ``formats`` beside native for each conversation started from now on; one already started keeps the + formats it started with, so no copy misses its first turns.""" + self._prescribed = frozenset(formats) - {self._native} + return self._prescribed def register(self, task_id: str, context_id: str) -> TrajectoryLog: """The task's log, created ``pending`` the first time the task is seen.""" @@ -132,7 +152,8 @@ def start(self, task_id: str, turn: TurnStart | None = None) -> None: started = sum(1 for other in order if self._tasks[other].started) order.insert(started, task_id) record.started = True - if self._atif is not None and turn is not None: + built = self._built.setdefault(record.context_id, self._prescribed) + if self._atif is not None and turn is not None and ATIF_FORMAT in built: copy = self._copies.get(record.context_id) if copy is None: copy = self._copies[record.context_id] = AtifCopy( @@ -153,6 +174,7 @@ def restored(self, context_id: str, turns: int) -> None: new document, since the conversation is the snapshot's now; an agent that serves atif/1 only.""" self._turns.set(context_id, turns) self._copies.pop(context_id, None) + self._built.pop(context_id, None) self.media.drop(context_id) def request_cancel(self, task_id: str) -> None: @@ -281,7 +303,7 @@ def atif_task(self, task_id: str) -> AtifTaskRead | None: """The ended task's atif/1 document as it stood when it ended, with its aux log and served turn context, or None when there is none.""" record = self._tasks.get(task_id) - copy = self._copies.get(record.context_id) if record is not None else None + copy = self._copy(record.context_id) if record is not None else None if copy is None: return None self._touch(record.context_id) @@ -290,7 +312,7 @@ def atif_task(self, task_id: str) -> AtifTaskRead | None: def atif_context(self, context_id: str) -> dict[str, Any] | None: """The context's atif/1 live page, or None when it has no closed step yet.""" - copy = self._copies.get(context_id) + copy = self._copy(context_id) if copy is None: return None self._touch(context_id) @@ -301,7 +323,7 @@ def atif_task_page( ) -> AtifCursor | None: """A cursor page of the task's closed atif/1 steps, in the task's state.""" record = self._tasks.get(task_id) - copy = self._copies.get(record.context_id) if record is not None else None + copy = self._copy(record.context_id) if record is not None else None if copy is None: return None self._touch(record.context_id) @@ -314,7 +336,7 @@ def atif_context_page( ) -> AtifCursor | None: """A cursor page of the context's closed atif/1 steps, on the task the context is on and in the context's state.""" - copy = self._copies.get(context_id) + copy = self._copy(context_id) records = self._context_records(context_id) if copy is not None else None if copy is None or records is None: return None @@ -322,6 +344,13 @@ def atif_context_page( page = copy.page(after, max_steps, max_bytes) return AtifCursor(page=page, context_id=context_id, task_id=task_id, state=state) + def _copy(self, context_id: str) -> AtifCopy | None: + """The context's atif/1 copy, None before it has one; NotPrescribed when it started without atif/1.""" + built = self._built.get(context_id) + if built is not None and ATIF_FORMAT not in built: + raise NotPrescribed(context_id, ATIF_FORMAT) + return self._copies.get(context_id) + def _context_progress( self, context_id: str, records: list[_TaskRecord] ) -> tuple[str, TrajectoryState]: @@ -362,6 +391,7 @@ def _evict(self, *, keep: str) -> None: continue del self._contexts[context_id] self._copies.pop(context_id, None) + self._built.pop(context_id, None) self.media.drop(context_id) for task_id in task_ids: del self._tasks[task_id] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index a3f8518e3..f551ce057 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -233,6 +233,19 @@ class TrajectoryMediaResponse(ExtensionResponse): failed: dict[str, str] = Field(default_factory=dict) +class TrajectoryPrescribeRequest(ExtensionRequest): + """The trajectory formats beside native the agent builds for each conversation it starts from now on, each one + its card's ``formats`` lists; none for native only, which it always builds.""" + + formats: list[str] = Field(default_factory=list) + + +class TrajectoryPrescribeResponse(ExtensionResponse): + """Every format the agent builds for a conversation it starts from now on: native first.""" + + formats: list[str] + + class AtifUploadedObjects(ExtensionResponse): trajectory: Uploaded aux: Uploaded | None = None @@ -1227,8 +1240,8 @@ def _configure_trajectory( return ExtensionConfiguration( wire_params={"formats": formats, "native": native, "default": native} if native else {}, options={"native": native, **({"atif": atif} if atif is not None else {})} if native else {}, - # An agent that serves atif/1 hands over its copies' media files, which the worker pulls at each turn's end. - features={"media"} if atif is not None else frozenset(), + # An agent that can build atif/1 builds it only when prescribed, and hands over its copies' media files. + features={"media", "prescribe"} if atif is not None else frozenset(), variants=( {"get": {"task_events", "context_events", "context", "context_objects"}} if live else {} ), @@ -1389,7 +1402,20 @@ def _configure_trajectory( sdk_only=True, ), } - ) + ), + "prescribe": OperationGroup( + operations={ + "prescribe": _op( + "prescribe", + "POST", + "/ext/trajectory/prescribe", + SDK, + request_model=TrajectoryPrescribeRequest, + response_model=TrajectoryPrescribeResponse, + sdk_only=True, + ), + } + ), }, configuration_validator=_configure_trajectory, ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py index cd1003913..ba3dca805 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/framework.py @@ -36,7 +36,7 @@ from ._atif_copy import TurnStart from ._secrets import Secrets from ._snapshot_state import SnapshotStates -from ._trajectory_log import MAX_TRAJECTORY_READ_BYTES, TaskTrajectories +from ._trajectory_log import MAX_TRAJECTORY_READ_BYTES, NotPrescribed, TaskTrajectories from ._triggers import TriggerEngine, TriggerError from .extensions import ( AGENT_CONFIG_V1, @@ -69,6 +69,8 @@ AtifUploadedObjects, TrajectoryEventsResponse, TrajectoryMediaRequest, + TrajectoryPrescribeRequest, + TrajectoryPrescribeResponse, TrajectoryObjectsResponse, TrajectoryState, TrajectoryUploadedObjects, @@ -147,6 +149,16 @@ def _config_wire_names(config: type[AgentConfig]) -> dict[str, str]: return result + +def _not_prescribed(exc: NotPrescribed) -> AgentEnvError: + return AgentEnvError( + "format_not_prescribed", + f"{exc.format} was not prescribed for context {exc.context_id!r} when it started, so the agent built none; " + "prescribe it before a conversation's first message (the trajectory extension's prescribe)", + status=409, + ) + + def _config_write_only_wire_names(config: type[AgentConfig]) -> set[str]: """Return wire names explicitly marked as write-only in the config schema.""" wire_names = _config_wire_names(config) @@ -684,6 +696,7 @@ def handlers(self) -> dict[tuple[str, str], Callable]: (SKILL_CONFIG_V1.uri, "list"): self.skill_list, (TRAJECTORY_V1.uri, "get"): self.trajectory_get, (TRAJECTORY_V1.uri, "media"): self.trajectory_media, + (TRAJECTORY_V1.uri, "prescribe"): self.trajectory_prescribe, (TRIGGERS_V1.uri, "register"): self.triggers_register, (TRIGGERS_V1.uri, "decide"): self.triggers_decide, (TRIGGERS_V1.uri, "state"): self.triggers_state, @@ -905,6 +918,8 @@ def _atif_events( request.context_id, request.after, max_steps, MAX_TRAJECTORY_READ_BYTES ) unknown = f"Unknown context '{request.context_id}'" + except NotPrescribed as exc: + raise _not_prescribed(exc) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc if cursor is None: @@ -934,7 +949,10 @@ async def _atif_document( through the request's grants, as the copy serves it, its secrets scrubbed.""" served: dict[str, Any] = {} if isinstance(request, (TaskTrajectoryRequest, TaskObjectTrajectoryRequest)): - read = self.task_trajectories.atif_task(request.task_id) + try: + read = self.task_trajectories.atif_task(request.task_id) + except NotPrescribed as exc: + raise _not_prescribed(exc) from exc if read is None: raise HTTPException( status_code=404, @@ -945,7 +963,10 @@ async def _atif_document( else: if isinstance(request, ContextObjectTrajectoryRequest): _refuse_turn_objects(request) - document = self.task_trajectories.atif_context(request.context_id) + try: + document = self.task_trajectories.atif_context(request.context_id) + except NotPrescribed as exc: + raise _not_prescribed(exc) from exc if document is None: raise HTTPException( status_code=404, @@ -966,6 +987,17 @@ async def _atif_document( ) return AtifObjectsResponse(objects=uploaded, **versions).model_dump(mode="json", exclude_none=True) + async def trajectory_prescribe(self, request: TrajectoryPrescribeRequest) -> dict[str, Any]: + """Build ``request.formats`` beside native for each conversation the agent starts from now on; each must be + one the card offers.""" + for format in request.formats: + self.check_trajectory_format(format) + built = self.task_trajectories.prescribe(request.formats) + native = self.trajectory_formats[0] + return TrajectoryPrescribeResponse( + formats=[native, *(format for format in self.trajectory_formats if format in built)] + ).model_dump(mode="json") + async def trajectory_media(self, request: TrajectoryMediaRequest) -> dict[str, Any]: """Upload the media files of a context's atif/1 copy the worker asks for, from the spool (``MediaSpool.push``).""" pushed = await self.task_trajectories.media.push(request.context_id, request.objects) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/client.py b/packages/agentenv-protocol/src/agentenv_protocol/client.py index 0d76ad4aa..2fa18a8d8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/client.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/client.py @@ -77,9 +77,28 @@ def request_headers(url: str, headers: dict[str, str] | None = None) -> dict[str def granted_headers(url: str, grants: dict[str | None, dict[str, str]]) -> dict[str, str]: """What ``grants``, env addresses and the headers that carry their control tokens, give a call to ``url``: those of - the address it is or is under, and none anywhere else.""" - return next((sent for base, sent in grants.items() - if base and (url == base.rstrip("/") or url.startswith(f"{base.rstrip('/')}/"))), {}) + the address it resolves to or under, and none anywhere else.""" + where = _resolved(url) + if where is None: + return {} + for base, sent in grants.items(): + address = _resolved(base) if base else None + if address is not None and where[0] == address[0] and where[1][:len(address[1])] == address[1]: + return sent + return {} + + +def _resolved(url: str) -> tuple[tuple[str, str, int | None], list[str]] | None: + """``url``'s origin and path segments as its server routes them, dot segments resolved; None for a relative url or + one whose path still holds a dot segment once percent-decoded, which a server may resolve out of its address.""" + try: + parsed = httpx.URL(url) + except httpx.InvalidURL: + return None + segments = parsed.path.rstrip("/").split("/") + if not parsed.is_absolute_url or any(segment in (".", "..") for segment in segments): + return None + return (parsed.scheme, parsed.host, parsed.port), segments async def _rpc(base_url: str, method: str, params: dict, timeout: int, verify: bool, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py index 213c81e76..28a568c24 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -6,7 +6,7 @@ from .keys import canonical from .operations import declared from .recording import check_recording -from .settings import ComputerSettings +from .settings import MAX_SCREENSHOT_DELAY_MS, ComputerSettings BUTTONS = ("left", "right", "middle", "back", "forward") @@ -32,6 +32,9 @@ def check_features(cls: type) -> None: value = getattr(cls, name) if not isinstance(value, int) or value < low: problems.append(f"{name} is an integer of at least {low}, got {value!r}") + delay = cls.screenshot_delay_ms + if not isinstance(delay, int) or not 0 <= delay <= MAX_SCREENSHOT_DELAY_MS: + problems.append(f"screenshot_delay_ms is an integer from 0 to {MAX_SCREENSHOT_DELAY_MS}, got {delay!r}") if cls.keys != "all": try: if [canonical(k) for k in cls.keys] != list(cls.keys): diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 4b35f438d..df40ae8f2 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -65,7 +65,8 @@ class DriverDeclaration(BaseModel): class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironment): """Subclass it and mark the methods that capture and send input (see ``operations``); the class is checked when - it is defined. Features are class attributes, on the card. A subclass's ``__init__`` calls ``super().__init__()``. + it is defined. Features are class attributes, on the card; ``screenshot_delay_ms``, how long an input's screenshot + waits, is a class attribute that isn't. A subclass's ``__init__`` calls ``super().__init__()``. """ buttons: ClassVar[tuple[str, ...]] = ("left",) @@ -78,6 +79,10 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen max_duration_ms: ClassVar[int] = 10_000 max_key_repeat: ClassVar[int] = 100 max_scroll_amount: ClassVar[int] = 100 + #: How long after an input ``screenshot: true`` captures, in ms: about the longest this driver's screens take to + #: show what an input did, as a smooth scroll or the page a key opens; a deploy's ``screenshot_delay_ms`` setting + #: replaces it. + screenshot_delay_ms: ClassVar[int] = 500 #: The (width, height) modes computer-display/v1 set takes; with ``@set_display`` only. display_modes: ClassVar[tuple] = () layouts: ClassVar[tuple[str, ...]] = ("us",) @@ -170,17 +175,20 @@ async def bundled_skill(self) -> tuple[str, str, dict[str, str]] | None: def create_app(self) -> Any: self._computer_require_init() - return self._computer_front_apps(super().create_app()) + return self._computer_finish_app(super().create_app()) def mount(self, app: Any) -> Any: self._computer_require_init() - return self._computer_front_apps(super().mount(app)) + return self._computer_finish_app(super().mount(app)) async def children_environments(self) -> list[EnvironmentCard] | None: """The computer's apps, from the folders it composed them into.""" return await apps.children(self._computer.apps) if self._computer.apps else None - def _computer_front_apps(self, app: Any) -> Any: + def _computer_finish_app(self, app: Any) -> Any: + """The app with the computer's apps' routes in front of them; it also logs the screenshot delay, once FastMCP has + set logging up.""" + _log.info("an input's screenshot is taken %d ms after it", self._computer.effective_screenshot_delay_ms()) app.custom_route("/svc/{name}/{path:path}", methods=svc.METHODS)(self._computer_forward) add_websocket_route(app, "/svc/{name}/{path:path}", self._computer_relay) for logger in svc.SERVER_LOGGERS: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 855b8b03a..d8e7d27aa 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -110,11 +110,12 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None, wrap: Callable[[InputResult, Any], InputResult] | None = None) -> CallToolResult: """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result - with others' input since ``frame`` and, when asked, a fresh frame. Whatever the call's turn can change is read - once it has the turn: a point ``mapped`` from ``frame`` is checked again, since the display may have changed - while it waited, and ``at_pointer`` reads the pointer and puts it before ``args``; input a page's open dialog - blocks is refused. ``then`` runs as soon as the input is delivered, and ``wrap`` makes the result from the - operation's own.""" + with others' input since ``frame`` and, when asked, a fresh frame, ``effective_screenshot_delay_ms()`` after the + input so it shows what the input did: a smooth scroll or the page a key opens renders a couple of hundred milliseconds + after the input returns. Whatever the call's turn can change is read once it has the turn: a point ``mapped`` + from ``frame`` is checked again, since the display may have changed while it waited, and ``at_pointer`` reads + the pointer and puts it before ``args``; input a page's open dialog blocks is refused. ``then`` runs as soon as + the input is delivered, and ``wrap`` makes the result from the operation's own.""" identity = self.driver.caller_identity(ctx) sent = False async with self.queue.turn(self.driver.settings.queue_wait_s): @@ -156,6 +157,7 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra result = wrap(result, effect) if not screenshot: return send(result) + await asyncio.sleep(self.effective_screenshot_delay_ms() / 1000) try: new = await self.capture(identity) except AgentEnvError as e: @@ -164,6 +166,12 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra result.summary, result.frame = f"{result.summary}, then captured {new.id}{new.note()}", new.result() return send(result, new.encode(), new.captured_at) + def effective_screenshot_delay_ms(self) -> int: + """How long after an input its screenshot is taken: the deploy's ``screenshot_delay_ms`` setting, else the + driver class's.""" + setting = self.driver.settings.screenshot_delay_ms + return self.driver.screenshot_delay_ms if setting is None else setting + async def _annotate(self, ctx: ToolContext, operation: str, args: tuple, move_to: tuple | None, repeat: int, hold: tuple | None, display: Display | None) -> None: """Mark an input on the recordings that draw them, as it is about to be delivered: one with a point on the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py index 5e4a629f4..8ec060cce 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/settings.py @@ -4,7 +4,7 @@ import json import os import re -from typing import Literal +from typing import Literal, Optional from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -28,6 +28,8 @@ def tool_prefix_problem(prefix: str) -> str | None: return None Space = Literal["frame", "norm"] +#: The longest ``screenshot_delay_ms`` a driver or a deploy may give, in ms. +MAX_SCREENSHOT_DELAY_MS = 10_000 class ComputerSettings(BaseModel): @@ -45,6 +47,8 @@ class ComputerSettings(BaseModel): drag_duration_ms: int = Field(default=500, ge=0) exec_output_bytes: int = Field(default=1 << 20, gt=0) exec_timeout_s: float = Field(default=60.0, gt=0, le=3600) + #: How long after an input ``screenshot: true`` captures, in ms; None for the driver's own ``screenshot_delay_ms``. + screenshot_delay_ms: Optional[int] = Field(default=None, ge=0, le=MAX_SCREENSHOT_DELAY_MS) def load_settings(environ: dict | None = None, model: type[ComputerSettings] = ComputerSettings) -> ComputerSettings: diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_media.py b/packages/agentenv-protocol/tests/test_a2a_atif_media.py index ab5cb980e..defc1d5cd 100644 --- a/packages/agentenv-protocol/tests/test_a2a_atif_media.py +++ b/packages/agentenv-protocol/tests/test_a2a_atif_media.py @@ -15,7 +15,7 @@ import pytest from starlette.testclient import TestClient -from test_a2a_atif_serving import _atif, _ended, _send +from test_a2a_atif_serving import _atif, _ended, _send, _serving from agentenv_protocol.a2a_agent import ( TRAJECTORY_V1, @@ -36,7 +36,17 @@ from agentenv_protocol.a2a_agent._media_spool import MediaSpool from agentenv_protocol.a2a_agent._atif_copy import TurnStart from agentenv_protocol.a2a_agent._trajectory_log import TaskTrajectories -from agentenv_protocol.atif import Adapter, AtifError, atif_adapter, check_media, errors_in, media_file, native, validate +from agentenv_protocol.atif import ( + FORMAT, + Adapter, + AtifError, + atif_adapter, + check_media, + errors_in, + media_file, + native, + validate, +) from agentenv_protocol.atif._image_size import image_size from agentenv_protocol.transfers import TransferError, WriteObject @@ -119,7 +129,7 @@ def _media(client: TestClient, paths: dict[str, str], context_id: str = "ctx-1") def test_a_copys_images_are_spooled_by_sha256_with_their_size_and_pushed_once_asked(store: dict[str, bytes]) -> None: - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png", "inbound.png")]) _ended(client, task_id) document = _atif(client, task_id=task_id)["trajectory"] @@ -144,7 +154,7 @@ def test_a_copys_images_are_spooled_by_sha256_with_their_size_and_pushed_once_as def test_a_media_calls_grants_leave_the_copys_media_paths_as_they_are(store: dict[str, bytes]) -> None: """A grant's URL may name the file it writes, ``media/.``: the scrub takes a URL's credential from it, never the file's name, which every read of the copy keeps.""" - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) _ended(client, task_id) assert _media(client, {_path(SHOT, ".png"): "image/png"}).status_code == 200 @@ -155,7 +165,7 @@ def test_a_media_calls_grants_leave_the_copys_media_paths_as_they_are(store: dic def test_the_same_bytes_added_twice_are_spooled_and_pushed_once(store: dict[str, bytes]) -> None: - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png"), _file(SHOT, "image/png")]) _ended(client, task_id) document = _atif(client, task_id=task_id)["trajectory"] @@ -177,7 +187,7 @@ async def upload(target, source) -> Uploaded: raise TransferError("grant_expired", "The transfer grant expired.") return await stored(target, source) - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png"), _file(other, "image/png")]) _ended(client, task_id) monkeypatch.setattr(_media_spool, "upload", upload) @@ -204,7 +214,7 @@ async def run(self, request: TaskRequest) -> TaskResult: async def trajectory(self, request: TaskTrajectoryRequest | TaskObjectTrajectoryRequest) -> dict[str, Any]: return {"trajectory": []} - with TestClient(OwnReads().create_app()) as client: + with _serving(OwnReads()) as client: _ended(client, _send(client, "look", "m-1", extra_parts=[_file(SHOT, "image/png")])) assert list(_media(client, {_path(SHOT, ".png"): "image/png"}).json()["objects"]) == [_path(SHOT, ".png")] @@ -241,7 +251,7 @@ async def upload(target, source) -> Uploaded: def test_a_spooled_file_that_no_longer_matches_its_name_never_reaches_the_store(store: dict[str, bytes]) -> None: - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) _spool(client).held("ctx-1", _path(SHOT, ".png")).write_bytes(b"changed on disk") pushed = _media(client, {_path(SHOT, ".png"): "image/png"}).json() @@ -253,7 +263,7 @@ def test_an_upload_whose_bytes_changed_in_transit_fails_and_its_file_stays_held( async def upload(target, source) -> Uploaded: return Uploaded(size_bytes=len(SHOT), sha256="0" * 64) - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) monkeypatch.setattr(_media_spool, "upload", upload) pushed = _media(client, {_path(SHOT, ".png"): "image/png"}).json() @@ -264,7 +274,7 @@ async def upload(target, source) -> Uploaded: def test_a_file_the_spool_cant_write_is_reported_missing_and_the_copy_goes_on( store: dict[str, bytes], tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: other = _png(2, 2) - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: spool = _spool(client) spool._folder = tmp_path / "removed" _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) @@ -283,26 +293,26 @@ def test_a_file_the_spool_cant_write_is_reported_missing_and_the_copy_goes_on( def test_a_media_call_takes_only_media_paths() -> None: - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: for path in ("../../etc/passwd", "media/abc.png", "https://elsewhere.example.test/x.png", f"media/{'a' * 64}/x.png"): refused = _media(client, {path: "image/png"}) assert refused.status_code == 400, (path, refused.text) -def test_only_an_agent_that_serves_atif_offers_media() -> None: +def test_only_an_agent_that_can_build_atif_offers_media_and_prescribe() -> None: def methods(agent: AgentEnvAgent) -> set[str]: card = TestClient(agent.create_app()).get("/.well-known/agent-card.json").json() [trajectory] = [ext for ext in card["capabilities"]["extensions"] if ext["uri"] == TRAJECTORY_V1.uri] return set(trajectory["params"]["methods"]) - assert methods(_shooting()) == {"get", "media"} + assert methods(_shooting()) == {"get", "media", "prescribe"} assert methods(_shooting(atif=False)) == {"get"} def test_a_video_and_a_heic_image_are_spooled_and_pushed_like_any_file(store: dict[str, bytes]) -> None: video, heic = b"\x00\x00\x00\x18ftypmp42" + b"v" * 64, b"\x00\x00\x00\x18ftypheic" + b"h" * 64 - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: task_id = _send(client, "shoot", "m-1", extra_parts=[_file(video, "video/mp4"), _file(heic, "image/heic")]) _ended(client, task_id) document = _atif(client, task_id=task_id)["trajectory"] @@ -318,6 +328,7 @@ def test_a_video_and_a_heic_image_are_spooled_and_pushed_like_any_file(store: di def test_a_context_evicted_or_restored_over_drops_its_media_but_what_another_holds() -> None: trajectories = TaskTrajectories(max_contexts=1, native=NATIVE, atif=Shots) + trajectories.prescribe([FORMAT]) def play(task_id: str, context_id: str, data: bytes) -> None: log = trajectories.register(task_id, context_id) @@ -338,7 +349,7 @@ def play(task_id: str, context_id: str, data: bytes) -> None: def test_the_spool_goes_with_the_server(store: dict[str, bytes]) -> None: - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: _ended(client, _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")])) folder = _spool(client).folder assert folder.is_dir() @@ -346,7 +357,7 @@ def test_the_spool_goes_with_the_server(store: dict[str, bytes]) -> None: def test_the_native_trajectory_is_unchanged_by_media(store: dict[str, bytes]) -> None: - with TestClient(_shooting().create_app()) as client: + with _serving(_shooting()) as client: task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) _ended(client, task_id) served = client.post("/ext/trajectory", json={"task_id": task_id}).json()["trajectory"] @@ -386,6 +397,7 @@ def test_an_images_size_is_read_from_its_header(data: bytes, media_type: str, si def test_dimensions_an_adapter_passes_win_over_the_header() -> None: trajectories = TaskTrajectories(native=NATIVE, atif=Shots) + trajectories.prescribe([FORMAT]) log = trajectories.register("t1", "ctx") trajectories.start("t1", TurnStart(message_id=None, parts=(TextPart(text="hi"),), agent_name="toy", agent_version="1", agent_model=None)) diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_prescribe.py b/packages/agentenv-protocol/tests/test_a2a_atif_prescribe.py new file mode 100644 index 000000000..fc7fca919 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_a2a_atif_prescribe.py @@ -0,0 +1,103 @@ +"""An agent builds the trajectory formats beside native that the run prescribes (``TRAJECTORY_V1.prescribe``), and only +those: with none prescribed it builds no atif/1 copy, spools no media and counts no turns, and its atif/1 reads are +refused. Each conversation keeps the formats it started with, so no copy misses its first turns.""" +from __future__ import annotations + +import base64 + +import pytest +from starlette.testclient import TestClient +from test_a2a_atif_media import SHOT, Shots, _file, _path, _shooting +from test_a2a_atif_serving import _ended, _read, _send, prescribe + +from agentenv_protocol.a2a_agent import TextPart, TrajectoryState +from agentenv_protocol.a2a_agent import _turn_index +from agentenv_protocol.a2a_agent._atif_copy import TurnStart +from agentenv_protocol.a2a_agent._trajectory_log import NotPrescribed, TaskTrajectories +from agentenv_protocol.atif import FORMAT + +NATIVE = "shots/1" + + +def _services(client: TestClient): + return client.app.state.agentenv_a2a.services + + +def _atif_reads(client: TestClient, task_id: str, context_id: str) -> list: + """Every atif/1 read of a task and its context: the task read, the context read and both cursor pages.""" + return [_read(client, format=FORMAT, task_id=task_id), _read(client, format=FORMAT, context_id=context_id), + _read(client, format=FORMAT, task_id=task_id, after=0), _read(client, format=FORMAT, context_id=context_id, + after=0)] + + +def test_an_agent_prescribed_nothing_builds_only_native_and_refuses_atif_reads() -> None: + with TestClient(_shooting().create_app()) as client: + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, task_id) + reads = _atif_reads(client, task_id, "ctx-1") + native = _read(client, task_id=task_id).json() + services = _services(client) + assert services.task_trajectories._copies == {} and services.task_trajectories.media.folder is None + assert [read.status_code for read in reads] == [409] * 4 + assert {read.json()["error"]["code"] for read in reads} == {"format_not_prescribed"} + assert native["format"] == NATIVE and native["trajectory"][0]["type"] == "shot" + assert not (_turn_index.turns_folder()).exists() or not any(_turn_index.turns_folder().iterdir()) + + +def test_a_conversation_keeps_the_formats_it_started_with_and_a_later_call_sets_the_next_ones() -> None: + with TestClient(_shooting().create_app()) as client: + before = _send(client, "shoot", "m-1", context_id="before") + _ended(client, before) + assert prescribe(client, FORMAT).json() == {"formats": [NATIVE, FORMAT]} + again = _send(client, "again", "m-2", context_id="before") + after = _send(client, "shoot", "m-3", context_id="after", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, again) + _ended(client, after) + assert prescribe(client).json() == {"formats": [NATIVE]} + later = _send(client, "shoot", "m-4", context_id="later") + _ended(client, later) + statuses = {name: _read(client, format=FORMAT, task_id=task).status_code + for name, task in (("before", again), ("after", after), ("later", later))} + held = _services(client).task_trajectories.media.held("after", _path(SHOT, ".png")) + assert statuses == {"before": 409, "after": 200, "later": 409} + assert held is not None + + +def test_a_prescription_names_only_formats_the_card_offers_and_may_name_native() -> None: + with TestClient(_shooting().create_app()) as client: + refused = prescribe(client, "elsewhere/2") + native_named = prescribe(client, NATIVE, FORMAT).json() + with TestClient(_shooting(atif=False).create_app()) as client: + unlisted = prescribe(client, FORMAT) + assert refused.status_code == 400 and refused.json()["error"]["code"] == "format_not_offered" + assert native_named == {"formats": [NATIVE, FORMAT]} + assert unlisted.status_code == 404 + + +def test_a_read_that_names_no_format_gets_native_whatever_is_prescribed() -> None: + with TestClient(_shooting().create_app()) as client: + prescribe(client, FORMAT) + task_id = _send(client, "shoot", "m-1", extra_parts=[_file(SHOT, "image/png")]) + _ended(client, task_id) + unnamed = _read(client, task_id=task_id).json() + assert unnamed["format"] == NATIVE + + +def test_a_conversation_restored_over_takes_the_prescription_in_effect_at_its_next_task() -> None: + trajectories = TaskTrajectories(native=NATIVE, atif=Shots) + + def play(task_id: str) -> None: + log = trajectories.register(task_id, "ctx") + trajectories.start(task_id, TurnStart(message_id=None, parts=(TextPart(text="hi"),), agent_name="toy", + agent_version="1", agent_model=None)) + log.append({"type": "shot", "step": "s", "data": base64.b64encode(SHOT).decode(), "media_type": "image/png"}) + trajectories.seal(task_id, TrajectoryState.COMPLETED, final=(TextPart(text="done"),)) + + play("t1") + with pytest.raises(NotPrescribed): + trajectories.atif_task("t1") + trajectories.prescribe([FORMAT]) + trajectories.restored("ctx", 1) + play("t2") + assert trajectories.atif_task("t2") is not None + trajectories.media.close() diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_scrubbing.py b/packages/agentenv-protocol/tests/test_a2a_atif_scrubbing.py index 9168ed816..7ee498e53 100644 --- a/packages/agentenv-protocol/tests/test_a2a_atif_scrubbing.py +++ b/packages/agentenv-protocol/tests/test_a2a_atif_scrubbing.py @@ -8,12 +8,13 @@ import random import re from datetime import datetime, timedelta, timezone +from contextlib import ExitStack from typing import Any, Callable from urllib.parse import quote import pytest from starlette.testclient import TestClient -from test_a2a_atif_serving import _ended, _send +from test_a2a_atif_serving import _ended, _send, _serving from agentenv_protocol.a2a_agent import ( CDP_CONFIG_V1, @@ -143,7 +144,7 @@ def _assert_redacted(reads: list[Any], secret: str) -> None: def test_a_secret_the_agent_registers_is_redacted_in_every_atif_read_and_kept_in_the_native_log( monkeypatch: pytest.MonkeyPatch) -> None: - with TestClient(_echoing(register=SECRET).create_app()) as client: + with _serving(_echoing(register=SECRET)) as client: task_id = _turn(client, f"the key is {SECRET}") reads = _reads(client, task_id, monkeypatch) native_read = _native(client, task_id) @@ -199,20 +200,11 @@ def _cdp_url(monkeypatch: pytest.MonkeyPatch, client: Callable[[], TestClient]) ids=["model-key", "model-headers", "control-token", "write-only-config", "mcp-headers", "cdp-url"]) def test_a_secret_the_sdk_holds_or_is_handed_is_redacted(monkeypatch: pytest.MonkeyPatch, handed) -> None: - app = None - - def client(**kwargs: Any) -> TestClient: - nonlocal app - app = _echoing().create_app() - return TestClient(app, **kwargs).__enter__() - - opened, secret = handed(monkeypatch, client) - try: + with ExitStack() as serving: + opened, secret = handed(monkeypatch, lambda **kwargs: serving.enter_context(_serving(_echoing(), **kwargs))) task_id = _turn(opened, f"echo: {secret}") reads = _reads(opened, task_id, monkeypatch) assert _native(opened, task_id) == [{"type": "say", "text": f"echo: {secret}"}] - finally: - opened.__exit__(None, None, None) _assert_redacted(reads, secret) @@ -249,7 +241,7 @@ def test_a_write_only_mapping_is_redacted_by_its_credentials_alone(monkeypatch: params = {"service_tier": "priority", "thinking": {"type": "enabled"}, "extra_headers": {"Authorization": "Bearer params-0123456789", "X-Title": "toy agent app"}, "auth": {"user": "bob-the-builder"}} - with caplog.at_level(logging.WARNING), TestClient(_echoing().create_app()) as client: + with caplog.at_level(logging.WARNING), _serving(_echoing()) as client: assert client.post("/ext/agent-config", json={"model_params": params}).status_code == 200 task_id = _turn(client, "priority enabled toy agent app Bearer params-0123456789 bob-the-builder") document = client.post("/ext/trajectory", json={"task_id": task_id, "format": FORMAT}).json()["trajectory"] @@ -324,7 +316,7 @@ def test_a_secret_is_redacted_in_mapping_keys_too() -> None: def test_a_non_ascii_secret_in_an_inbound_data_part_is_redacted(monkeypatch: pytest.MonkeyPatch) -> None: secret = 'pä"ss-wörd-0123' - with TestClient(_echoing(register=secret).create_app()) as client: + with _serving(_echoing(register=secret)) as client: task_id = _send(client, "hello", "m-1", extra_parts=[{"kind": "data", "data": {"password": secret}}]) _ended(client, task_id) reads = _reads(client, task_id, monkeypatch) @@ -392,7 +384,7 @@ def test_a_cursor_page_counts_its_bytes_as_it_serves_them() -> None: def test_an_agent_that_serves_no_atif_registers_nothing_and_registering_before_its_app_is_refused() -> None: - with TestClient(_echoing(register=SECRET, atif=False).create_app()) as client: + with _serving(_echoing(register=SECRET, atif=False)) as client: task_id = _turn(client, f"the key is {SECRET}") assert _native(client, task_id) == [{"type": "say", "text": f"the key is {SECRET}"}] with pytest.raises(RuntimeError, match="after create_app"): diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_serving.py b/packages/agentenv-protocol/tests/test_a2a_atif_serving.py index a79b9fc02..016fd1691 100644 --- a/packages/agentenv-protocol/tests/test_a2a_atif_serving.py +++ b/packages/agentenv-protocol/tests/test_a2a_atif_serving.py @@ -7,6 +7,8 @@ import json import threading import time +from collections.abc import Iterator +from contextlib import contextmanager from datetime import datetime, timedelta, timezone from typing import Any, Awaitable, Callable @@ -290,6 +292,22 @@ def _read(client: TestClient, **payload: Any) -> Any: return client.post("/ext/trajectory", json=payload) +def prescribe(client: TestClient, *formats: str) -> Any: + return client.post("/ext/trajectory/prescribe", json={"formats": list(formats)}) + + +@contextmanager +def _serving(agent: Any, **kwargs: Any) -> Iterator[TestClient]: + """The agent served in-process with atif/1 prescribed when its card offers it, as a run that reads its atif/1 + copies prescribes it before a conversation's first message.""" + with TestClient(agent.create_app(), **kwargs) as client: + card = client.get("/.well-known/agent-card.json").json() + [trajectory] = [ext for ext in card["capabilities"]["extensions"] if ext["uri"] == TRAJECTORY_V1.uri] or [{}] + if "prescribe" in (trajectory.get("params") or {}).get("methods", {}): + assert prescribe(client, FORMAT).status_code == 200 + yield client + + def _atif(client: TestClient, **payload: Any) -> dict[str, Any]: response = _read(client, format=FORMAT, **payload) assert response.status_code == 200, response.text @@ -342,7 +360,7 @@ async def capture(_target: Any, body: bytes) -> Uploaded: monkeypatch.setattr(framework, "upload", capture) native_bodies = [] for agent in (_toy(atif=None), _toy()): - with TestClient(agent.create_app()) as client: + with _serving(agent) as client: task_id = _send(client, "work t1", "m-1") _ended(client, task_id) bodies = [_read(client, **read).json() for read in ( @@ -351,7 +369,7 @@ async def capture(_target: Any, body: bytes) -> Uploaded: native_bodies.append(json.loads(json.dumps(bodies).replace(task_id, "TASK"))) # Another agent's container, whose turn counts aren't the first one's. monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "another-container")) - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: task_id = _send(client, "work t1", "m-1") _ended(client, task_id) task = _atif(client, task_id=task_id) @@ -386,7 +404,7 @@ async def capture(_target: Any, body: bytes) -> Uploaded: ("work", "completed"), ("failed", "failed"), ("raised", "failed"), ("canceled", "canceled")]) def test_the_copy_validates_on_every_terminal_path(scenario: str, state: str) -> None: hold = _Hold() - with TestClient(_toy(hold).create_app()) as client: + with _serving(_toy(hold)) as client: task_id = _send(client, f"{scenario} t1", "m-1") if scenario == "canceled": hold.reached() @@ -405,7 +423,7 @@ def test_the_copy_validates_on_every_terminal_path(scenario: str, state: str) -> def test_two_tasks_in_one_context_make_one_document() -> None: - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: first = _send(client, "work t1", "m-1") _ended(client, first) second = _send(client, "work t2", "m-2") @@ -426,7 +444,7 @@ def test_two_tasks_in_one_context_make_one_document() -> None: def test_a_step_waiting_on_its_result_holds_back_the_steps_after_it() -> None: hold = _Hold() pages = [] - with TestClient(_toy(hold).create_app()) as client: + with _serving(_toy(hold)) as client: task_id = _send(client, "late t1", "m-1") hold.reached(1) pages.append(_atif(client, context_id="ctx-1")["trajectory"]) @@ -454,7 +472,7 @@ async def capture(target: Any, body: bytes) -> Uploaded: monkeypatch.setattr(framework, "upload", capture) grants = {name: {**_grant(), "write": {**_grant()["write"], "url": f"https://objects.example.test/{name}"}} for name in ("trajectory", "aux", "turn_context")} - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: tasks = [] for tag in ("t1", "t2"): tasks.append(_send(client, f"hooks {tag}", f"m-{tag}")) @@ -484,7 +502,7 @@ async def capture(target: Any, body: bytes) -> Uploaded: def test_an_atif_context_read_goes_to_the_sdk_even_when_the_agent_binds_its_own() -> None: - with TestClient(_toy(bind_context=True).create_app()) as client: + with _serving(_toy(bind_context=True)) as client: task_id = _send(client, "work t1", "m-1") _ended(client, task_id) own = _read(client, context_id="ctx-1").json() @@ -495,7 +513,7 @@ def test_an_atif_context_read_goes_to_the_sdk_even_when_the_agent_binds_its_own( def test_data_parts_are_written_as_the_conversation_or_the_profile() -> None: image = base64.b64encode(b"\x89PNG fake").decode() - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: task_id = _send(client, "data t1", "m-1", extra_parts=[ {"kind": "data", "data": {"ticket": 42, "priority": "high"}}, {"kind": "file", "file": {"name": "shot.png", "mimeType": "image/png", "bytes": image}}, @@ -516,7 +534,7 @@ def test_data_parts_are_written_as_the_conversation_or_the_profile() -> None: def test_an_adapter_agent_that_appends_and_returns_fails_the_turn() -> None: - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: task_id = _send(client, "both t1", "m-1") _ended(client, task_id, "failed") answer = client.post("/a2a", json={"jsonrpc": "2.0", "id": "g", "method": "tasks/get", @@ -527,7 +545,7 @@ def test_an_adapter_agent_that_appends_and_returns_fails_the_turn() -> None: def test_a_returned_native_trajectory_runs_through_the_adapter_when_the_task_ends() -> None: - with TestClient(_toy(live=False).create_app()) as client: + with _serving(_toy(live=False)) as client: task_id = _send(client, "returned t1", "m-1") _ended(client, task_id) document = _atif(client, task_id=task_id)["trajectory"] @@ -538,7 +556,7 @@ def test_a_returned_native_trajectory_runs_through_the_adapter_when_the_task_end def test_an_adapter_error_stops_the_copy_not_the_turn() -> None: - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: task_id = _send(client, "broken t1", "m-1") _ended(client, task_id) native = _read(client, task_id=task_id) @@ -552,7 +570,7 @@ def test_an_adapter_error_stops_the_copy_not_the_turn() -> None: def test_cursor_pages_carry_the_header_and_their_tasks_sub_agents() -> None: - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: task_id = _send(client, "spawn t1", "m-1") _ended(client, task_id) document = _atif(client, task_id=task_id)["trajectory"] @@ -575,7 +593,7 @@ def test_cursor_pages_carry_the_header_and_their_tasks_sub_agents() -> None: def test_a_reader_following_the_cursor_gets_the_steps_a_sub_agent_writes_after_its_result() -> None: hold = _Hold() pages = [] - with TestClient(_toy(hold).create_app()) as client: + with _serving(_toy(hold)) as client: task_id = _send(client, "grow t1", "m-1") hold.reached() while not pages or pages[-1]["has_more"]: @@ -594,7 +612,7 @@ def test_a_reader_following_the_cursor_gets_the_steps_a_sub_agent_writes_after_i def test_a_spawn_its_task_ended_without_answering_keeps_its_sub_agent_referenced() -> None: hold = _Hold() - with TestClient(_toy(hold).create_app()) as client: + with _serving(_toy(hold)) as client: task_id = _send(client, "orphan t1", "m-1") hold.reached() client.post("/a2a", json={"jsonrpc": "2.0", "id": "c", "method": "tasks/cancel", "params": {"id": task_id}}) @@ -608,7 +626,7 @@ def test_a_spawn_its_task_ended_without_answering_keeps_its_sub_agent_referenced def test_an_unknown_or_unfinished_read_is_not_found() -> None: hold = _Hold() - with TestClient(_toy(hold).create_app()) as client: + with _serving(_toy(hold)) as client: task_id = _send(client, "canceled t1", "m-1") hold.reached() unfinished = _read(client, task_id=task_id, format=FORMAT) @@ -621,7 +639,7 @@ def test_an_unknown_or_unfinished_read_is_not_found() -> None: def test_two_messages_sent_at_once_in_one_context_are_two_tasks_of_one_document() -> None: # The second waits on the context's lock; the first is closed in the copy before that lock lets the second start. - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: first = _send(client, "work t1", "m-1") second = _send(client, "work t2", "m-2") _ended(client, first) @@ -637,7 +655,7 @@ def test_an_agent_on_the_envs_clock_stamps_the_sdks_steps_with_the_time_its_prog clock_file.write_text(f"@1748768400.000000000 x1 ^{time.time()}\n") monkeypatch.setattr(follower, "CLOCK_FILE", str(clock_file)) monkeypatch.setattr(agent_clock, "follows", lambda: True) - with TestClient(_toy().create_app()) as client: + with _serving(_toy()) as client: task_id = _send(client, "data t1", "m-1") _ended(client, task_id) answer = _atif(client, task_id=task_id) diff --git a/packages/agentenv-protocol/tests/test_a2a_atif_turns.py b/packages/agentenv-protocol/tests/test_a2a_atif_turns.py index 5645ff1e4..46dc1c12a 100644 --- a/packages/agentenv-protocol/tests/test_a2a_atif_turns.py +++ b/packages/agentenv-protocol/tests/test_a2a_atif_turns.py @@ -12,7 +12,7 @@ import pytest from starlette.testclient import TestClient -from test_a2a_atif_serving import NATIVE, Hooks, _atif, _ended, _Hold, _send, _state, _turn +from test_a2a_atif_serving import _atif, _ended, _Hold, Hooks, NATIVE, _send, _serving, _state, _turn from agentenv_protocol.a2a_agent import ( SNAPSHOT_STATE_PARAM, @@ -37,7 +37,7 @@ from agentenv_protocol.a2a_agent import _snapshot_state, _turn_index from agentenv_protocol.a2a_agent._atif_copy import TurnStart from agentenv_protocol.a2a_agent._trajectory_log import TaskTrajectories -from agentenv_protocol.atif import errors_in, validate +from agentenv_protocol.atif import FORMAT, errors_in, validate _IDENTITY = AgentIdentity(name="toy-agent", description="test", version="2.0") @@ -109,13 +109,13 @@ async def download(source, destination: Path) -> None: def test_a_restarted_server_goes_on_counting_in_a_new_document_of_the_same_session() -> None: - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: for tag in ("t1", "t2"): task_id = _send(client, f"work {tag}", f"m-{tag}") _ended(client, task_id) before = _atif(client, task_id=task_id)["trajectory"] # The same container's home, a new process. - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: task_id = _send(client, "work t3", "m-t3") _ended(client, task_id) after = _atif(client, task_id=task_id)["trajectory"] @@ -128,6 +128,7 @@ def test_a_restarted_server_goes_on_counting_in_a_new_document_of_the_same_sessi def test_a_context_evicted_from_memory_goes_on_counting() -> None: trajectories = TaskTrajectories(max_contexts=1, native=NATIVE, atif=Hooks) + trajectories.prescribe([FORMAT]) def play(task_id: str, context_id: str) -> None: log = trajectories.register(task_id, context_id) @@ -148,7 +149,7 @@ def play(task_id: str, context_id: str) -> None: def test_a_snapshot_carries_the_turn_count_to_the_fresh_agent_it_is_restored_into( objects: dict[str, bytes], monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: for tag in ("t1", "t2"): _ended(client, _send(client, f"work {tag}", f"m-{tag}")) saved = _save(client) @@ -157,7 +158,7 @@ def test_a_snapshot_carries_the_turn_count_to_the_fresh_agent_it_is_restored_int assert json.loads(objects["state"]) == {"turns": 2} # Another container: none of the first one's counts. monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "fresh-agent")) - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: loaded = _load(client, "ctx-restored") assert loaded.status_code == 200, loaded.text task_id = _send(client, "work t3", "m-t3", context_id="ctx-restored") @@ -169,7 +170,7 @@ def test_a_snapshot_carries_the_turn_count_to_the_fresh_agent_it_is_restored_int def test_a_restore_into_a_context_the_agent_holds_starts_a_new_document(objects: dict[str, bytes]) -> None: - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: _ended(client, _send(client, "work t1", "m-t1")) _save(client) task_id = _send(client, "work t2", "m-t2") @@ -187,13 +188,13 @@ def test_a_restore_into_a_context_the_agent_holds_starts_a_new_document(objects: def test_a_snapshot_state_that_isnt_the_sdks_is_refused(objects: dict[str, bytes]) -> None: objects["state"] = b'{"turns": -1}' - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: refused = _load(client, "ctx-restored") assert refused.status_code == 400 and refused.json()["error"]["code"] == "snapshot_state_invalid", refused.text def test_an_agent_that_serves_no_atif_writes_no_state(objects: dict[str, bytes]) -> None: - with TestClient(_snapshotting(atif=None).create_app()) as client: + with _serving(_snapshotting(atif=None)) as client: saved = _save(client) assert saved.status_code == 200, saved.text assert "state" not in saved.json()["objects"] and "state" not in objects @@ -211,7 +212,7 @@ def snapshot_params(agent: AgentEnvAgent) -> dict[str, Any]: def test_a_task_waits_for_its_contexts_save_so_the_saved_count_is_its_captures(objects: dict[str, bytes]) -> None: paused = _Hold() - with TestClient(_snapshotting(save_hold=paused).create_app()) as client: + with _serving(_snapshotting(save_hold=paused)) as client: for tag in ("t1", "t2"): _ended(client, _send(client, f"work {tag}", f"m-{tag}")) with concurrent.futures.ThreadPoolExecutor(1) as pool: @@ -233,7 +234,7 @@ def test_a_task_waits_for_its_contexts_save_so_the_saved_count_is_its_captures(o def test_a_load_into_a_context_waits_for_its_save_so_the_save_keeps_its_count(objects: dict[str, bytes]) -> None: objects["older-state"] = b'{"turns": 0}' paused = _Hold() - with TestClient(_snapshotting(save_hold=paused).create_app()) as client: + with _serving(_snapshotting(save_hold=paused)) as client: for tag in ("t1", "t2"): _ended(client, _send(client, f"work {tag}", f"m-{tag}")) with concurrent.futures.ThreadPoolExecutor(2) as pool: @@ -254,7 +255,7 @@ def test_a_load_into_a_context_waits_for_its_save_so_the_save_keeps_its_count(ob def test_a_save_while_a_task_runs_is_answered_without_waiting_for_it(objects: dict[str, bytes]) -> None: running = _Hold() - with TestClient(_snapshotting(run_hold=running).create_app()) as client: + with _serving(_snapshotting(run_hold=running)) as client: _ended(client, _send(client, "work t1", "m-t1")) task_id = _send(client, "late t2", "m-t2") running.reached() @@ -299,7 +300,7 @@ def test_a_count_that_cant_be_read_starts_again_with_a_warning(tmp_path: Path, c def test_a_snapshot_state_grant_names_json() -> None: - with TestClient(_snapshotting().create_app()) as client: + with _serving(_snapshotting()) as client: refused = _save(client, state_type="application/octet-stream") assert refused.status_code == 400 and "state media_type" in refused.text diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index c447ff478..680c738f3 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -119,6 +119,8 @@ def test_unknown_predicates_are_refused(): ({"keys": ("Return",)}, "keys: 'Return' is not a W3C key value"), ({"display_modes": ((10, 10),)}, "display_modes and @set_display come together"), ({"screen_kind": "tablet"}, "screen_kind is desktop or page, got 'tablet'"), + ({"screenshot_delay_ms": -1}, "screenshot_delay_ms is an integer from 0 to 10000, got -1"), + ({"screenshot_delay_ms": 0.5}, "screenshot_delay_ms is an integer from 0 to 10000, got 0.5"), ]) def test_features_are_checked_when_the_class_is_defined(features, error): with pytest.raises(TypeError, match=error): diff --git a/packages/agentenv-protocol/tests/test_computer_screenshot_delay.py b/packages/agentenv-protocol/tests/test_computer_screenshot_delay.py new file mode 100644 index 000000000..5031d4dd9 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_screenshot_delay.py @@ -0,0 +1,129 @@ +"""An input's screenshot is taken ``screenshot_delay_ms`` after the input, so it shows what the input did: a browser's +smooth scroll or the page a key opens renders a couple of hundred milliseconds after the input returns.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +import asyncio # noqa: E402 +import base64 # noqa: E402 +import io # noqa: E402 +import logging # noqa: E402 +import time # noqa: E402 + +from computer_fake import BACKGROUND, FakeDriver, mcp_session # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol.computer import ComputerSettings, scroll # noqa: E402 + +RED = (255, 0, 0) + + +class LateScreen(FakeDriver): + """A fake whose scroll reaches the screen ``lag`` seconds after the scroll returns, filling it red.""" + + def __init__(self, lag: float = 0.1, **kwargs) -> None: + super().__init__(**kwargs) + self.lag = lag + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers): + await super().scroll_at(x, y, direction, amount, modifiers) + asyncio.get_running_loop().call_later(self.lag, self.fill) + + def fill(self) -> None: + self.screen.image = PILImage.new("RGB", self.screen.image.size, RED) + + +class Instant(LateScreen): + """A driver whose screens show an input at once, so it takes the screenshot at once.""" + + screenshot_delay_ms = 0 + + +def _colour(result) -> tuple: + [image] = [c for c in result.content if c.type == "image"] + return PILImage.open(io.BytesIO(base64.b64decode(image.data))).convert("RGB").getpixel((10, 10)) + + +async def _scroll(session, screenshot: bool = True): + return await session.call_tool("computer_scroll", {"x": 5, "y": 5, "direction": "down", "screenshot": screenshot}) + + +@pytest.mark.asyncio +async def test_the_screenshot_shows_a_change_that_reaches_the_screen_after_the_input(): + driver = LateScreen() + assert driver.settings.screenshot_delay_ms is None and driver.screenshot_delay_ms == 500 + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + started = time.monotonic() + scrolled = await _scroll(session) + took = time.monotonic() - started + assert _colour(scrolled) == RED + assert took >= 0.5 + assert scrolled.structuredContent["frame"]["frame_id"] == "f2" + + +@pytest.mark.asyncio +async def test_with_no_delay_the_screenshot_is_taken_at_once(): + async with mcp_session(LateScreen(lag=2.0, settings=ComputerSettings(screenshot_delay_ms=0))) as session: + await session.call_tool("computer_screenshot", {}) + scrolled = await _scroll(session) + assert _colour(scrolled) == BACKGROUND + + +@pytest.mark.asyncio +async def test_the_driver_sets_its_own_delay(): + async with mcp_session(Instant(lag=2.0)) as session: + await session.call_tool("computer_screenshot", {}) + scrolled = await _scroll(session) + assert _colour(scrolled) == BACKGROUND + + +@pytest.mark.asyncio +async def test_a_deploys_setting_replaces_the_drivers_delay(): + async with mcp_session(Instant(settings=ComputerSettings(screenshot_delay_ms=300))) as session: + await session.call_tool("computer_screenshot", {}) + scrolled = await _scroll(session) + assert _colour(scrolled) == RED + + +@pytest.mark.asyncio +async def test_the_driver_logs_the_delay_it_uses(caplog): + with caplog.at_level(logging.INFO, logger="agentenv_protocol.computer.driver"): + async with mcp_session(Instant(settings=ComputerSettings(screenshot_delay_ms=300))): + pass + assert "an input's screenshot is taken 300 ms after it" in caplog.messages + + +@pytest.mark.asyncio +async def test_an_input_without_a_screenshot_doesnt_wait(): + async with mcp_session(LateScreen(settings=ComputerSettings(screenshot_delay_ms=5000))) as session: + await session.call_tool("computer_screenshot", {}) + started = time.monotonic() + scrolled = await _scroll(session, screenshot=False) + took = time.monotonic() - started + assert not scrolled.isError and "frame" not in scrolled.structuredContent + assert took < 2.5 + + +@pytest.mark.asyncio +async def test_the_wait_before_the_screenshot_doesnt_hold_other_input(): + driver = LateScreen(settings=ComputerSettings(screenshot_delay_ms=1000)) + finished = [] + + async def call(name: str, tool: str, args: dict) -> None: + await session.call_tool(tool, args) + finished.append(name) + + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + scrolling = asyncio.ensure_future(call("scroll", "computer_scroll", + {"x": 5, "y": 5, "direction": "down", "screenshot": True})) + await asyncio.sleep(0.2) + await call("click", "computer_click", {"x": 7, "y": 7}) + await scrolling + assert finished == ["click", "scroll"] + assert [entry[0] for entry in driver.screen.log] == ["scroll", "click"] diff --git a/packages/agentenv-protocol/tests/test_control_token.py b/packages/agentenv-protocol/tests/test_control_token.py index 88f42ffbd..5474f9fbb 100644 --- a/packages/agentenv-protocol/tests/test_control_token.py +++ b/packages/agentenv-protocol/tests/test_control_token.py @@ -129,6 +129,24 @@ def test_an_agents_token_travels_in_its_own_header_not_authorization(): assert client.control_token_headers(None) == {} and client.control_token_headers("") == {} +@pytest.mark.parametrize("url, granted", [ + ("http://proxy/sandbox/A/svc/mcp-x/agentenv", True), + ("http://PROXY:80/sandbox/A/./svc/x", True), + ("http://proxy/sandbox/A/svc/x/../y", True), + ("http://proxy/sandbox/A/svc/mcp-x/../../../B/x", False), + ("http://proxy/sandbox/A/../B/x", False), + ("http://proxy/sandbox/A/%2e%2e/B/x", False), + ("http://proxy/sandbox/AB/x", False), + ("https://proxy/sandbox/A/x", False), + ("../sandbox/A/x", False), +]) +def test_a_grant_covers_only_urls_that_resolve_under_its_address(url, granted): + """The server resolves dot segments before it routes, so a dotted path that climbs out of the env's address + reaches another env behind the same proxy.""" + assert client.granted_headers(url, {"http://proxy/sandbox/A/": client.control_headers("s3cret")}) == ( + {"Authorization": "Bearer s3cret"} if granted else {}) + + @pytest.mark.asyncio async def test_invoke_extension_and_the_data_plane_send_the_token(monkeypatch): seen = [] diff --git a/pyproject.toml b/pyproject.toml index 472b5cc94..acf347af9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agentenv-framework" -version = "0.9.1603" +version = "0.9.1609" description = "SDK and CLI for building, deploying and running agent environments and the tasks that grade agents in them." readme = "README.md" license = "Apache-2.0" diff --git a/src/agent_env/a2a_agent/object_transfer.py b/src/agent_env/a2a_agent/object_transfer.py index c1a2aea34..5ae20a70d 100644 --- a/src/agent_env/a2a_agent/object_transfer.py +++ b/src/agent_env/a2a_agent/object_transfer.py @@ -659,13 +659,35 @@ def to(cls, store: ObjectStore, object_url: str) -> TrajectoryUpload: return cls(object_url, write) +@dataclass(frozen=True) +class TrajectoryFormats: + """The trajectory formats an agent's card offers: every one it serves, its native one and the one its reads + serve when they name none. A card from before agents named their formats offers none, and its reads refuse a + ``format``, so ``native`` is None for it too.""" + + formats: tuple[str, ...] = () + native: str | None = None + default: str | None = None + + @classmethod + def of(cls, extension: Mapping[str, Any] | None) -> TrajectoryFormats: + """The formats the trajectory extension's card params offer.""" + params = (extension or {}).get("params") or {} + formats = params.get("formats") + if not isinstance(formats, list) or not formats: + return cls() + return cls(tuple(formats), params.get("native"), params.get("default")) + + @dataclass(frozen=True) class FetchedTrajectory: """Where a trajectory ``get`` left the trajectory: uploaded by the agent through the grant - (``object_url``) or in the answer (``inline``). Both None: the agent returned none.""" + (``object_url``) or in the answer (``inline``). Both None: the agent returned none. ``format`` + is the one the answer says it served, None when it says none.""" object_url: str | None = None inline: Any = None + format: str | None = None async def fetch_trajectory( @@ -676,10 +698,14 @@ async def fetch_trajectory( timeout: float | None = None, store: ObjectStore | None = None, headers: Mapping[str, str] | None = None, + format: str | None = None, ) -> FetchedTrajectory: - """POST one trajectory ``get``. With ``upload`` the agent uploads the trajectory through its - grant, issued by ``store``, and the answer is checked. The default wait covers that upload.""" + """POST one trajectory ``get``, naming ``format`` when given: only to a card that offers formats + (``TrajectoryFormats.of``). With ``upload`` the agent uploads the trajectory through its grant, + issued by ``store``, and the answer is checked. The default wait covers that upload.""" payload: dict[str, Any] = dict(selector) + if format is not None: + payload["format"] = format if upload is not None: objects = TrajectoryWriteObjects(trajectory=upload.write) # Only the grants given: a pinned agent's request model forbids the optional ones newer SDKs take. @@ -698,7 +724,8 @@ async def fetch_trajectory( headers=headers, ) if upload is not None: - return FetchedTrajectory(object_url=upload.object_url) + return FetchedTrajectory(object_url=upload.object_url, format=body.format) if not isinstance(body, Mapping): return FetchedTrajectory() - return FetchedTrajectory(inline=body.get("trajectory")) + served = body.get("format") + return FetchedTrajectory(inline=body.get("trajectory"), format=served if isinstance(served, str) else None) diff --git a/src/agent_env/a2a_agent/trajectory_follower.py b/src/agent_env/a2a_agent/trajectory_follower.py index 5162727ee..daf20e0df 100644 --- a/src/agent_env/a2a_agent/trajectory_follower.py +++ b/src/agent_env/a2a_agent/trajectory_follower.py @@ -68,20 +68,23 @@ async def follow_trajectory( stop: asyncio.Event, client: httpx.AsyncClient | None = None, headers: Mapping[str, str] | None = None, + format: str | None = None, ) -> None: """Read task ``task_id``'s trajectory from its first event, handing ``sink`` each page that has - events and the last one, until the task has ended and every event is read. + events and the last one, until the task has ended and every event is read. Each read names + ``format`` when given: only to a card that offers formats. Between reads it waits ``poll_interval_seconds``, or less once ``stop`` is set. Transport errors and 5xx answers are retried with backoff until ``stop`` is set, and end following after it; a task the agent no longer knows (404) ends following.""" after = 0 failures = 0 + named = {"format": format} if format is not None else {} async with nullcontext(client) if client is not None else httpx.AsyncClient() as http: while True: try: response = await http.post( - endpoint, json={"task_id": task_id, "after": after}, headers=headers, + endpoint, json={"task_id": task_id, "after": after, **named}, headers=headers, timeout=_READ_TIMEOUT_SECONDS, ) if response.status_code == 404: diff --git a/src/agent_env/agents/agents/codex/README.md b/src/agent_env/agents/agents/codex/README.md index ace3dad2c..c01142ad0 100644 --- a/src/agent_env/agents/agents/codex/README.md +++ b/src/agent_env/agents/agents/codex/README.md @@ -17,7 +17,9 @@ Codex reads its settings from `$CODEX_HOME/config.toml`, which the agent writes: - At start, `model_providers.litellm` points Codex at `$LITELLM_BASE_URL/openai/v1` with the Responses API, its key from `LITELLM_API_KEY`, `[model.headers]` (`LITELLM_EXTRA_HEADERS`) on every model call, and a run's `project_id` and `task_id` as `x-litellm-customer-id` and `x-litellm-tags`. The run's tags join an `x-litellm-tags` that - `[model.headers]` names, whose own keys, such as `taskId`, win. + `[model.headers]` names, whose own keys, such as `taskId`, win. Its `capabilities.remote_compaction = "v2"` makes + Codex compact through the endpoint, with the turn's tools declared, instead of locally. `tools.update_plan.enabled` + is set too, since Codex leaves `update_plan` off by default. - Before each task, every registered MCP server, the task's `peer_agents` tools included, as an `mcp-remote` stdio server. Its `AgentEnv-Role` header is the role its registration names, unless the task has a role of its own, which reaches that task's CLI alone as `-c` overrides, so tasks of different roles running at once share the file. Codex @@ -54,7 +56,7 @@ tools it names, and a list to turn off those whose tools it names. A list names | `create_goal`, `get_goal`, `update_goal` | `features.goals=false` | | `web_search` | `web_search="disabled"` | -No setting turns off `apply_patch`: a model's entry in Codex's model catalog offers it (every GPT model 0.147.0 ships +No setting turns off `apply_patch`: a model's entry in Codex's model catalog offers it (every GPT model 0.162.1 ships with does). So a selection without it (`{"disable": "*"}`, a list naming it, or an `{"only": [...]}` that doesn't) writes a copy of the catalog the CLI would use, with `apply_patch_tool_type` null for every model, to `$CODEX_HOME/model_catalog_without_apply_patch.json`, and points the task's CLI at it with `-c model_catalog_json=...`. diff --git a/src/agent_env/agents/agents/codex/codex_config.py b/src/agent_env/agents/agents/codex/codex_config.py index e60d0f358..3ba4467c7 100644 --- a/src/agent_env/agents/agents/codex/codex_config.py +++ b/src/agent_env/agents/agents/codex/codex_config.py @@ -32,7 +32,7 @@ _TRUST_ENV = ("NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE") # The CLI version CodexTool names the tools of; a test holds it to package.json's @openai/codex. -TOOLS_CLI_VERSION = "0.147.0" +TOOLS_CLI_VERSION = "0.162.1" # The CLI's own tools, by the names its model sees, that a selection can turn off. Some only go together, as one # setting turns them on: the shell's two, the six collaboration tools `multi_agent_v2` brings, and the three goal tools. # `apply_patch` is turned off through the model catalog (`catalog_overrides`), the others by `_TOOL_SWITCHES`. Left out: @@ -90,7 +90,7 @@ def builtin_tools_overrides(builtin_tools: CodexTools | None) -> list[str]: # No setting turns off `apply_patch`: a model's entry in the CLI's model catalog offers it when its -# `apply_patch_tool_type` is set, as it is for every GPT model 0.147.0 ships with. A selection without it points the CLI, +# `apply_patch_tool_type` is set, as it is for every GPT model 0.162.1 ships with. A selection without it points the CLI, # with `model_catalog_json`, at a copy of the catalog it would use whose models all have the field null. The copy is # made from the installed CLI's own catalog each time, so it follows the pinned version. CATALOG_WITHOUT_APPLY_PATCH = "model_catalog_without_apply_patch.json" @@ -210,6 +210,8 @@ def write_base_config(environ: Mapping[str, str]) -> None: # Ships default-OFF, and the default-ON `multi_agent` is v1 scaffolding that # yields no collaboration tools on codex-family models. cfg.setdefault("features", {})["multi_agent_v2"] = True + # Off by default since 0.162.0; a selection turns it off again with a `-c` override. + cfg.setdefault("tools", {}).setdefault("update_plan", {})["enabled"] = True base_url = environ.get("LITELLM_BASE_URL") or "" if base_url: cfg["model_provider"] = "litellm" @@ -236,6 +238,10 @@ def write_base_config(environ: Mapping[str, str]) -> None: if name.lower() != "x-litellm-tags"} if headers: provider["http_headers"] = headers + # Codex compacts a custom provider's context locally by default, with a request that declares no tools, which + # a Responses API may refuse when the history replays encrypted tool-call arguments. Remote compaction sends + # the tools. + provider["capabilities"] = {"remote_compaction": "v2"} cfg.setdefault("model_providers", {})["litellm"] = provider _write(cfg) diff --git a/src/agent_env/agents/agents/codex/package-lock.json b/src/agent_env/agents/agents/codex/package-lock.json index 335fc10b9..60ee349d9 100644 --- a/src/agent_env/agents/agents/codex/package-lock.json +++ b/src/agent_env/agents/agents/codex/package-lock.json @@ -5,14 +5,14 @@ "packages": { "": { "dependencies": { - "@openai/codex": "0.147.0", + "@openai/codex": "0.162.1", "mcp-remote": "0.14.3" } }, "node_modules/@openai/codex": { - "version": "0.147.0", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0.tgz", - "integrity": "sha512-EQLEXecAG2ptxI7UpBMo2TR/ga5596/c/OsYF/0LoUDh5JANZ7IoGqlzBEWbuEVQ76JePIbtTW/ihCkp1a7Z3w==", + "version": "0.162.1", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1.tgz", + "integrity": "sha512-NWZdi/kxyjv/8EUGFupziGU38YyleugZRM4JXgY5XFH7FUmaFA33NZS2Bmq0HPazf7S3jJQQWsZ/jAK9jsrV3Q==", "license": "Apache-2.0", "bin": { "codex": "bin/codex.js" @@ -21,19 +21,19 @@ "node": ">=16" }, "optionalDependencies": { - "@openai/codex-darwin-arm64": "npm:@openai/codex@0.147.0-darwin-arm64", - "@openai/codex-darwin-x64": "npm:@openai/codex@0.147.0-darwin-x64", - "@openai/codex-linux-arm64": "npm:@openai/codex@0.147.0-linux-arm64", - "@openai/codex-linux-x64": "npm:@openai/codex@0.147.0-linux-x64", - "@openai/codex-win32-arm64": "npm:@openai/codex@0.147.0-win32-arm64", - "@openai/codex-win32-x64": "npm:@openai/codex@0.147.0-win32-x64" + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.162.1-darwin-arm64", + "@openai/codex-darwin-x64": "npm:@openai/codex@0.162.1-darwin-x64", + "@openai/codex-linux-arm64": "npm:@openai/codex@0.162.1-linux-arm64", + "@openai/codex-linux-x64": "npm:@openai/codex@0.162.1-linux-x64", + "@openai/codex-win32-arm64": "npm:@openai/codex@0.162.1-win32-arm64", + "@openai/codex-win32-x64": "npm:@openai/codex@0.162.1-win32-x64" } }, "node_modules/@openai/codex-darwin-arm64": { "name": "@openai/codex", - "version": "0.147.0-darwin-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0-darwin-arm64.tgz", - "integrity": "sha512-BEUVkiOW7kLcRyrMLfAr/h9wF8sRVJyZDy6OHtVn6QGDXiv3BvAZVTY1Pu9xF7KdIdkYXbp4uayN0aDQQaAUJw==", + "version": "0.162.1-darwin-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1-darwin-arm64.tgz", + "integrity": "sha512-YVXpg57itBuQxjofay3KDF5UbnZF4Y1ZmNFzUCz0ZQiQN+DndxOjN5V87rnXvPaC4qCPO2El74DzewoBP8JyPQ==", "cpu": [ "arm64" ], @@ -48,9 +48,9 @@ }, "node_modules/@openai/codex-darwin-x64": { "name": "@openai/codex", - "version": "0.147.0-darwin-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0-darwin-x64.tgz", - "integrity": "sha512-Tb8McE5SvJIH0Vs5R6sq7u+quiC931yan2KOOl6km1OdZ82+Wi7eF5XrSFPs5CF7xCgoIK4Vs+byMbT5hN+ZUw==", + "version": "0.162.1-darwin-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1-darwin-x64.tgz", + "integrity": "sha512-0/WHRhbUnM1wAjfyCO54Q8s4vQRxLVBbqW6XuQf7rZTEMTYPKRIQU9aPeRgjG653ZCSjtBrslWQ+G9urY9CksQ==", "cpu": [ "x64" ], @@ -65,9 +65,9 @@ }, "node_modules/@openai/codex-linux-arm64": { "name": "@openai/codex", - "version": "0.147.0-linux-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0-linux-arm64.tgz", - "integrity": "sha512-SLC1JXw2TYfr/c3HhrJubyyLelq7vTOLWVmiThFA+z0+WgzCPmaseJ/kzDD3Gge/TO7fCnnj7UcPmC0d2c8XAg==", + "version": "0.162.1-linux-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1-linux-arm64.tgz", + "integrity": "sha512-Ss7W/SrKtQFNaMtSojNoiykJgX+iDC8daM+YrcvgmJADtQPk0cd3w6Dna4CYH2CYvf9nQranj69fpcTzZ3sQIQ==", "cpu": [ "arm64" ], @@ -82,9 +82,9 @@ }, "node_modules/@openai/codex-linux-x64": { "name": "@openai/codex", - "version": "0.147.0-linux-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0-linux-x64.tgz", - "integrity": "sha512-0W9MBxPpWW0cSkNqrTDN2jR7rzzT7oNMhQY5446lT2Lw5cz5yhDTck4Va9rjkQEm+HlFzP/dmEMSZbXfJsINmw==", + "version": "0.162.1-linux-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1-linux-x64.tgz", + "integrity": "sha512-cjK2mdHIfEepZ1lQ1MX3aTQC+8ncIsvnacl/6vI2dIIcfP8/MrhTQ2+qRlkXFNDzJSj1d36quIszP+3/4CUJWQ==", "cpu": [ "x64" ], @@ -99,9 +99,9 @@ }, "node_modules/@openai/codex-win32-arm64": { "name": "@openai/codex", - "version": "0.147.0-win32-arm64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0-win32-arm64.tgz", - "integrity": "sha512-e2ZstJ8zT8Rm1nvR7CUVO+Gr3cTChE41+VfOzGhynzDXEoW0wfbjUQbc2bWbh1arG94LMm4y3dqBtUIbSrfeGA==", + "version": "0.162.1-win32-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1-win32-arm64.tgz", + "integrity": "sha512-teVVACnEcupkR6XSmUQKwlGg/oA15d1DQDPVSoAfilHgctPEpTi5y11jYS6JhJO+AFnYM84gyo5SOS5vlnCQSg==", "cpu": [ "arm64" ], @@ -116,9 +116,9 @@ }, "node_modules/@openai/codex-win32-x64": { "name": "@openai/codex", - "version": "0.147.0-win32-x64", - "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.147.0-win32-x64.tgz", - "integrity": "sha512-oT7Ss5fAPf2fiWE9QNURqZcQGAAawSVxmIUdgPzckq4KFZAM+pRz9JbM4Rr498CjtbNgTOjWvDJ+DXvIBSfOPA==", + "version": "0.162.1-win32-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.162.1-win32-x64.tgz", + "integrity": "sha512-dfl4MhHuEi0ARKyNmrTxk+RQmsWNCQAoVLUtrYq+yrW1C8P2sg5ziwgHUUACHjYLKuACV7ARH5Nq5pnCz3GlaQ==", "cpu": [ "x64" ], diff --git a/src/agent_env/agents/agents/codex/package.json b/src/agent_env/agents/agents/codex/package.json index 39f407faa..34cf6eefd 100644 --- a/src/agent_env/agents/agents/codex/package.json +++ b/src/agent_env/agents/agents/codex/package.json @@ -1,7 +1,7 @@ { "private": true, "dependencies": { - "@openai/codex": "0.147.0", + "@openai/codex": "0.162.1", "mcp-remote": "0.14.3" } } diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index e1e8d9695..b07c1ef1c 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -82,6 +82,11 @@ def task_file_parts(data: Any) -> tuple[list[dict], list[str] | None]: headlines is None or isinstance(headlines, list) and all(isinstance(h, str) for h in headlines)): raise ValueError("a task file must be a JSON array of step objects, or an object whose steps is one and whose " "headline_verifiers lists step ids") + for index, step in enumerate(steps): + for key in ("id", "type"): + if isinstance(step.get(key), (list, dict)): + raise ValueError(f"step #{index}'s {key} is a JSON {'array' if isinstance(step[key], list) else 'object'}" + f"; it must be a string") return steps, headlines diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index f8accd6e3..57d5f29e2 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -57,6 +57,7 @@ from agent_env.store.base import NotFoundError from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.routing import namespace_routing_enabled +from agent_env.task.lint import Finding, lint, refuses from agent_env.task.task import Task from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep @@ -198,6 +199,9 @@ class Plan: alias_uses: Mapping[EntityKind, Mapping[str, AliasUse]] = field(default_factory=dict) agent: AgentOverride | None = None # the agent --agent names for the solver variant: Varied | None = None # what --set changes, written under its variant's ids + # Each selected task's task lint findings that didn't refuse it, with its path: its warnings, and its errors too + # when the plan skipped lint. + lint_findings: tuple[tuple[str, Finding], ...] = () def bundle_of(self, entry: BundleEntry) -> Bundle: """The bundle ``entry`` comes from: the installed one for an entry pulled in from one, else the run's.""" @@ -210,14 +214,18 @@ def path(self, entry: BundleEntry) -> str: def plan_bundle(resolved: ResolvedBundle, *, tasks: Sequence[str] = (), evals: Sequence[str] = (), - installed: Installed | None = None, agent: str | None = None, default_agents: bool = True) -> Plan: + installed: Installed | None = None, agent: str | None = None, default_agents: bool = True, + skip_lint: bool = False, lint: bool = True) -> Plan: """Plan a run of ``tasks`` and ``evals``, each a name or id in this bundle. With neither, the run is every eval or, in a bundle without evals, every task. ``installed`` lists the installed bundles whose entities a reference pulls in (``installed_bundles``); without it, an id under one is a store id. ``agent`` names the agent the solver runs as, as a task's reference would, leaving the tasks as written (``Plan.agent``). Without ``default_agents``, a step that leaves its agent unset pulls in no configured default, as for a task written to - run wherever another's default applies. Raises BundleError listing every problem found.""" - return _Planner(resolved, installed, default_agents).plan(tasks, evals, agent) + run wherever another's default applies. With ``skip_lint``, the errors of task lint's own rules are findings it + keeps (``Plan.lint_findings``), not problems; a step's config problems stay problems. Without ``lint``, task lint + doesn't run, for a caller that only selects tasks and checks those it keeps. Raises BundleError listing every + problem found.""" + return _Planner(resolved, installed, default_agents, skip_lint, lint).plan(tasks, evals, agent) def plan_entities(resolved: ResolvedBundle, ids: Sequence[str], *, installed: Installed | None = None) -> Plan: @@ -244,10 +252,14 @@ class _Pullable: class _Planner: - def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, default_agents: bool = True): + def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, default_agents: bool = True, + skip_lint: bool = False, lint: bool = True): self.resolved = resolved self.default_agents = default_agents + self.skip_lint, self.lint = skip_lint, lint self.problems: list[str] = [] + self.lint_findings: list[tuple[str, Finding]] = [] + self.lint_refused = 0 self.nodes: dict[_Key, ResolvedEntry | BundleImage] = {} self.rank: dict[_Key, int] = {} self.edges: dict[_Key, list[_Key]] = {} @@ -575,7 +587,7 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr store_refs, store_latest = self._check_store_refs(closure) if override is not None: override = replace(override, version=store_latest.get((EntityKind.AGENT, override.agent_id))) - self._build_tasks(closure) + self._build_tasks(closure, linted=self.lint) self._check_files(closure) self._check_cli_loads(closure) self._check_env_types(closure) @@ -583,11 +595,15 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr self._check_environment_names(closure, store_latest) self._check_loaded_environments(closure, store_latest) if self.problems: - raise NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems) + refused = NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems) + if self.lint_refused: + refused.add_note(f"task lint found {self.lint_refused} of these; --skip-lint runs the tasks anyway") + raise refused writes = self._ordered(closure) return Plan(self.resolved, writes, tuple(selected_tasks), tuple(selected_evals), store_refs, store_latest, dict(self.origins), self._published(writes), - {kind: dict(uses) for kind, uses in self.alias_uses.items()}, override) + {kind: dict(uses) for kind, uses in self.alias_uses.items()}, override, + lint_findings=tuple(self.lint_findings)) def check(self) -> None: """The checks a run makes before it reads a store. A bundle with nothing to run passes only when its @@ -1122,8 +1138,12 @@ def _store_entity(self, ref: Reference, store_latest: dict[tuple[EntityKind, str except (NotFoundError, ValueError, KeyError, TypeError): return None # reported by the store ref check - def _build_tasks(self, closure: set[_Key]) -> None: - """Build each task's steps the way a write will, so a bad step fails before anything is written.""" + def _build_tasks(self, closure: set[_Key], *, linted: bool = False) -> None: + """Build each task's steps the way a write will, so a bad step fails before anything is written. ``linted``, + for a run or registration, adds task lint: its errors are problems, unless the plan skips lint, and the rest + ``lint_findings``. The + store-free check leaves it out, since a folder ``bundle init`` writes leaves user-scoped fields for its runner + to fill in.""" registry = get_task_step_registry() for key in sorted(closure, key=self.rank.__getitem__): source = self.nodes[key] @@ -1140,10 +1160,17 @@ def _build_tasks(self, closure: set[_Key]) -> None: self.steps[key] = steps if not failed: try: - Task(id=source.entry.id, version=None, steps=steps, - headline_verifiers=source.entry.headline_verifiers) + task = Task(id=source.entry.id, version=None, steps=steps, + headline_verifiers=source.entry.headline_verifiers) except ValueError as e: self.problems.append(f"{self._path(source.entry)}: {e}") + continue + for finding in lint(task, source.config, unpinned=False) if linted else (): + if refuses(finding, skip_lint=self.skip_lint): + self.problems.append(f"{self._path(source.entry)}: {finding.text}") + self.lint_refused += not refuses(finding, skip_lint=True) + else: + self.lint_findings.append((self._path(source.entry), finding)) # Order diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index bb8b75551..d9b401619 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -82,6 +82,7 @@ def register_bundle( on_progress: Callable[[str], None] | None = None, sandbox: str | None = None, emulate: bool = True, + skip_lint: bool = False, ) -> Registration: """Register the bundle at ``root``: ``tasks`` and ``evals`` by name, or with neither every task and eval it has, or every entity when it has none, with what they reference. ``prefix`` opens each entry's registry id, the folder's @@ -91,7 +92,8 @@ def register_bundle( run of the folder would (``declaring_for``). ``sandbox`` is the provider its tasks deploy on, as a run's is: each computer env composes only the forms its deploys there boot, and without it every form its base declares. Either way it composes on the env's provider, emulating an arch no link of that one runs natively unless ``emulate`` is - False, which leaves it skipped. + False, which leaves it skipped. ``skip_lint`` registers tasks task lint's own rules + refuse (``Registration.plan.lint_findings``). It turns namespace routing on while it runs. Raises BundleError listing every problem found before anything is written; a write that fails stops it, and every earlier one stays, in the ledger, for the next run to reuse. So does @@ -101,7 +103,7 @@ def register_bundle( with namespace_routing(): bundle = parse_bundle(Path(root)) resolved = resolve_bundle(bundle) - plan = _plan(resolved, tasks, evals) + plan = _plan(resolved, tasks, evals, skip_lint) ids, problems = registry_ids(plan, _prefix(bundle, prefix)) if problems: raise BundleError(problems) @@ -130,7 +132,7 @@ def registered_line(plan: Plan, done: Materialized) -> str: return f"{_label(plan, done.write)} -> {done.write.kind.store} {done.write.id} v{done.version} ({why})" -def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str]) -> Plan: +def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str], skip_lint: bool = False) -> Plan: """The plan of ``tasks`` and ``evals``, or of every task and eval, or of every entity of a bundle with none.""" entries = resolved.bundle.entries if not tasks and not evals: @@ -138,7 +140,8 @@ def _plan(resolved: ResolvedBundle, tasks: Sequence[str], evals: Sequence[str]) evals = [entry.name for entry in entries if entry.kind is BundleKind.EVAL] if not tasks and not evals: return plan_entities(resolved, [entry.id for entry in entries], installed=installed_bundles) - return plan_bundle(resolved, tasks=tasks, evals=evals, installed=installed_bundles, default_agents=False) + return plan_bundle(resolved, tasks=tasks, evals=evals, installed=installed_bundles, default_agents=False, + skip_lint=skip_lint) def _prefix(bundle: Bundle, prefix: str | None) -> str: @@ -241,8 +244,9 @@ def _slug(name: str) -> str: # The fields of a Plan that hold no ``@local`` id, so a translated plan keeps them: the store's latest versions, by store -# id, and the agent ``--agent`` names, which a registration plans without. -_KEPT = frozenset({"store_latest", "agent", "variant"}) +# id, the agent ``--agent`` names, which a registration plans without, and task lint's warnings, which name the bundle's +# files. +_KEPT = frozenset({"store_latest", "agent", "variant", "lint_findings"}) def translated(plan: Plan, ids: Mapping[str, str]) -> Plan: diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 15707cbe6..df39f0f49 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -32,6 +32,7 @@ from agent_env.store.routing import namespace_routing, run_scope from agent_env.task.store import record_task_cancelled from agent_env.task.interrupts import Interrupts +from agent_env.task.lint import said from agent_env.utils.progress import run_progress from agent_env.task.task import Task, recorded_scores from agent_env.task.teardown import TeardownReport, attempt_epoch, record_teardown, teardown_run @@ -225,6 +226,7 @@ def run_bundle( emulate: bool = True, source: HubSource | None = None, variant: Variant | None = None, + skip_lint: bool = False, ) -> BundleRun: """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the @@ -237,7 +239,8 @@ def run_bundle( (``InstalledBundle.id_root``) or one downloaded from the Hub (``HubBundle.id_root``); a folder's ids otherwise come from its path. ``source`` says where a Hub bundle came from (``HubBundle.source``), which each run's context records under ``metadata["bundle_source"]``. ``variant`` sets env keys for this run (``with_variant``), writing - what it changes under ids of their own and leaving the bundle as written. + what it changes under ids of their own and leaving the bundle as written. ``skip_lint`` runs tasks task lint's own + rules refuse, saying why (``plan_bundle``). Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. @@ -264,7 +267,7 @@ def run_bundle( with namespace_routing(): prepared = _prepare(root, tasks, evals, sandbox, id_root, agent, False, say, declaring_for(Path(root), installed=id_root is not None, dry_run=False), composing, - variant=variant) + variant=variant, skip_lint=skip_lint) plan, entries = prepared.plan, [entry for entry, _ in prepared.tasks] with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) @@ -296,6 +299,7 @@ def prepare_bundle( played: bool = False, agent: str | None = None, solver_endpoint: bool = False, + skip_lint: bool = False, ) -> PreparedBundle: """Do what ``run_bundle`` with these arguments does before its first task runs, and stop there. ``played`` leaves out what each task's only prompted agent would deploy, for a caller that plays that agent itself @@ -307,7 +311,7 @@ def prepare_bundle( ``run_bundle`` raises before its first task runs, except for a running event loop, which it doesn't need.""" composing = _composing(sandbox) return _prepare(root, tasks, evals, sandbox, id_root, agent, played, _progress(on_progress), _hint(id_root), - composing, solver_endpoint=solver_endpoint) + composing, solver_endpoint=solver_endpoint, skip_lint=skip_lint) def _write_bundle( @@ -352,6 +356,7 @@ def dry_run_bundle( id_root: str | None = None, emulate: bool = True, variant: Variant | None = None, + skip_lint: bool = False, ) -> DryRun: """Check what ``run_bundle`` with these arguments checks before its first task runs, and say what it would write and run, writing and running nothing: no entity, ledger row, lock or instance. Each write's version is @@ -365,7 +370,7 @@ def dry_run_bundle( composing = _composing(sandbox, emulate) say = _progress(on_progress) with namespace_routing(): - plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant) + plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, variant=variant, skip_lint=skip_lint) _say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True)) materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)), composing=composing, sandbox=sandbox, @@ -408,9 +413,9 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandbox: str | None, id_root: str | None, agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring, composing: Composing = Composing(), *, solver_endpoint: bool = False, - variant: Variant | None = None) -> PreparedBundle: + variant: Variant | None = None, skip_lint: bool = False) -> PreparedBundle: plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played, - solver_endpoint=solver_endpoint, variant=variant) + solver_endpoint=solver_endpoint, variant=variant, skip_lint=skip_lint) _say_plan(plan, say, declaring) materialization = _written_with_infra(plan, preflight, say, composing, sandbox) to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan)) @@ -419,14 +424,15 @@ def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandb def _planned(root: Path | str, tasks: Sequence[str], evals: Sequence[str], id_root: str | None, sandbox: str | None, agent: str | None = None, *, played: bool = False, - solver_endpoint: bool = False, variant: Variant | None = None) -> tuple[Plan, Preflight]: + solver_endpoint: bool = False, variant: Variant | None = None, + skip_lint: bool = False) -> tuple[Plan, Preflight]: bundle = parse_bundle(Path(root), id_root=id_root) varied = with_variant(bundle, variant) if variant is not None else None if varied is not None: tasks, evals = varied.selected(tasks, BundleKind.TASK), varied.selected(evals, BundleKind.EVAL) resolved = resolve_bundle(varied.bundle if varied is not None else bundle) plan = plan_bundle(without_played_agents(resolved) if played else resolved, tasks=tasks, evals=evals, - installed=installed_bundles, agent=agent) + installed=installed_bundles, agent=agent, skip_lint=skip_lint) refuse_store_tasks(plan) if varied is not None: plan = replace(plan, variant=varied) @@ -447,10 +453,11 @@ def _composing(sandbox: str | None, emulate: bool = True) -> Composing: def planned_tasks(root: Path | str, *, tasks: Sequence[str] = (), evals: Sequence[str] = (), agent: str | None = None) -> tuple[ResolvedEntry, ...]: - """The tasks ``run_bundle`` with these arguments would run, as resolved, without checking or writing anything. - Namespace routing must be on, as for ``plan_bundle``.""" + """The tasks ``run_bundle`` with these arguments would run, as resolved, without checking or writing anything, + task lint included: a caller that keeps some checks those as it prepares them. Namespace routing must be on, as + for ``plan_bundle``.""" plan = plan_bundle(resolve_bundle(parse_bundle(Path(root))), tasks=tasks, evals=evals, installed=installed_bundles, - agent=agent) + agent=agent, lint=False) selected = set(_to_run(plan)) return tuple(write.source for write in plan.writes if write.kind is BundleKind.TASK and write.source.entry in selected) @@ -484,8 +491,8 @@ def refuse_store_tasks(plan: Plan) -> None: def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> None: """Say what each agent, env and artifact alias names, what the solver deploys in place of the tasks' agents, the - ids what --set changes is written under, and what plugin packages the bundle uses that its bundle.toml doesn't - require, as ``declaring`` says (``declare``).""" + ids what --set changes is written under, what plugin packages the bundle uses that its bundle.toml doesn't + require, as ``declaring`` says (``declare``), and its tasks' lint findings, each kind once.""" for use in (use for uses in plan.alias_uses.values() for use in uses.values()): say(_aliased(use)) if plan.agent is not None: @@ -498,6 +505,8 @@ def _say_plan(plan: Plan, say: Callable[[str], None], declaring: Declaring) -> N say(f"{variant.typed}: {_changed(plan)} written under ids ending in ({variant.label})") for line in declare(plan.bundle, declaring): say(line) + for line in said(plan.lint_findings): + say(line) def _changed(plan: Plan) -> str: diff --git a/src/agent_env/cli/bundle.py b/src/agent_env/cli/bundle.py index b33996231..cd8031c33 100644 --- a/src/agent_env/cli/bundle.py +++ b/src/agent_env/cli/bundle.py @@ -13,6 +13,7 @@ from agent_env.bundle.requires import Declaring, UnmetRequirements, declaring_for from agent_env.cli._install_prompt import offer_to_install from agent_env.cli.utils import interruptible +from agent_env.task.lint import said from agent_env.entity.scopes import FieldScope @@ -94,8 +95,10 @@ def _report(imported: BundleImport, folder: Path) -> None: @click.option("--no-emulate", is_flag=True, help="Skip, rather than emulate, a computer env's arch the provider it composes on doesn't run " "natively; a later register or env build builds it.") +@click.option("--skip-lint", is_flag=True, + help="Register tasks agent-env task lint refuses for its own rules (a field no step reads, a headline or aggregate source that grades nothing), saying them; a step's own config problems still refuse it.") def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefix: str | None, - dry_run: bool, yes: bool, sandbox: str | None, no_emulate: bool): + dry_run: bool, yes: bool, sandbox: str | None, no_emulate: bool, skip_lint: bool): """Write the bundle at FOLDER into the configured stores under registry ids, so whatever runs the stores' tasks runs what the folder runs. @@ -113,7 +116,8 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi whoever wrote it. A store other than this machine's is written only after you confirm, or with --yes. Registering again writes only what changed since. A computer env whose compose leaves no build a deploy can boot fails the registration once every entity is written, before any task: registering again composes it again.""" - options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox, "emulate": not no_emulate} + options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox, "emulate": not no_emulate, + "skip_lint": skip_lint} click.echo(f"Registering {folder} into:") for line in describe_target(): click.echo(f" {line}") @@ -126,6 +130,8 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi if not dry_run: offer_to_install(e) raise + for line in said(planned.plan.lint_findings): + click.echo(f" {line}") for line in planned.floating: click.echo(f" floating, since its field takes no version: {line}") for line in planned.requires: diff --git a/src/agent_env/cli/eval/run.py b/src/agent_env/cli/eval/run.py index cbba1698f..56de0569d 100644 --- a/src/agent_env/cli/eval/run.py +++ b/src/agent_env/cli/eval/run.py @@ -35,7 +35,9 @@ def _write_context(context, task_id, output_dir, prefix=""): @click.option("--user-model", default=None, type=str, help="Override the model the user simulator runs on") @click.option("--judge-model", default=None, type=str, help="Override the model rubrics_verifier judges grade with") @click.option("--agent-artifact-id", default=None, type=str, help="Override the agent image artifact for deploy_agent steps") -def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, max_concurrency: int | None, agent_model: str | None, user_model: str | None, judge_model: str | None, agent_artifact_id: str | None): +@click.option("--skip-lint", is_flag=True, + help="Run tasks agent-env task lint finds errors in, saying them, as eval run did before it linted") +def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, max_concurrency: int | None, agent_model: str | None, user_model: str | None, judge_model: str | None, agent_artifact_id: str | None, skip_lint: bool): """Run all tasks in an eval through the configured [runner], concurrently.""" if k < 1: raise click.BadParameter("must be at least 1", param_hint="'--k'") @@ -44,6 +46,7 @@ def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, from agent_env.eval import Eval from agent_env.task import Task + from agent_env.task.lint import UNPINNED, lint, refuses, said, unpinned_summary click.echo(f"Fetching eval: id={eval_id} version={eval_version or 'latest'}...") eval_obj = Eval.get(eval_id, version=eval_version) @@ -56,6 +59,17 @@ def run(eval_id: str, eval_version: int | None, output_dir: str | None, k: int, click.echo(f" Loaded task: {task.id} v{task.version} ({len(task.steps)} steps){version_note}") tasks.append(task) + findings = [(f"{task.id} v{task.version}", finding) for task in tasks for finding in lint(task)] + refused = [] if skip_lint else [(task, f) for task, f in findings if refuses(f)] + if refused: + click.echo(f"Task lint found {len(refused)} error(s), so no run started (--skip-lint runs anyway):", err=True) + for task, finding in refused: + click.echo(f" - {task}: {finding.text}", err=True) + raise SystemExit(1) + for line in [*said((task, f) for task, f in findings if f.rule != UNPINNED), unpinned_summary(findings)]: + if line is not None: + click.echo(f" {line}") + print_banner() total_runs = len(tasks) * k diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 294afaf30..c69644fe6 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -56,10 +56,12 @@ @click.option("--no-emulate", is_flag=True, help="Skip, rather than emulate, a computer env's arch the provider it composes on doesn't run " "natively; a later run or env build builds it.") +@click.option("--skip-lint", is_flag=True, + help="Run tasks agent-env task lint refuses for its own rules (a field no step reads, a headline or aggregate source that grades nothing), saying them; a step's own config problems still refuse it.") @click.pass_context def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: tuple[str, ...], model: str | None, user_model: str | None, judge_model: str | None, agent: str | None, settings: tuple[str, ...], - sandbox: str | None, keep: bool, dry_run: bool, no_emulate: bool): + sandbox: str | None, keep: bool, dry_run: bool, no_emulate: bool, skip_lint: bool): """Run a bundle's tasks and evals, or list the installed bundles. BUNDLE is a folder, the name of a bundle an installed package provides (PACKAGE/NAME when several @@ -103,9 +105,9 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t """ if bundle is None: if tasks or evals or model or user_model or judge_model or agent or settings or sandbox or keep or dry_run \ - or no_emulate: + or no_emulate or skip_lint: raise click.UsageError("--task, --eval, --model, --user-model, --judge-model, --agent, --set, --sandbox, " - "--keep, --dry-run and --no-emulate need a BUNDLE to run") + "--keep, --dry-run, --no-emulate and --skip-lint need a BUNDLE to run") _list_installed() return verbose = ctx.find_root().params.get("verbose") @@ -117,12 +119,12 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t click.echo(_DRY_RUN) _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, agent=agent, on_progress=click.echo, id_root=id_root, emulate=not no_emulate, - variant=variant)) + variant=variant, skip_lint=skip_lint)) return result = run_bundle(root, tasks=tasks, evals=evals, model=model, user_model=user_model, judge_model=judge_model, agent=agent, sandbox=sandbox, on_progress=click.echo, id_root=id_root, keep=keep, emulate=not no_emulate, source=hub.source if hub else None, - variant=variant) + variant=variant, skip_lint=skip_lint) except BundleError as e: if isinstance(e, NotInStore) and hub is not None and hub.setup_note: e.add_note(hub.setup_note) diff --git a/src/agent_env/cli/task/__init__.py b/src/agent_env/cli/task/__init__.py index 41541af66..e99bd7b44 100644 --- a/src/agent_env/cli/task/__init__.py +++ b/src/agent_env/cli/task/__init__.py @@ -3,6 +3,7 @@ from .create import create from .get import get from .get_instance import get_instance +from .lint import lint from .run import run, run_batch from .teardown import teardown from .validate import validate @@ -17,6 +18,7 @@ def task(): task.add_command(create) task.add_command(get) task.add_command(get_instance) +task.add_command(lint) task.add_command(run) task.add_command(run_batch) task.add_command(teardown) diff --git a/src/agent_env/cli/task/create.py b/src/agent_env/cli/task/create.py index 455dc5312..3b033e43b 100644 --- a/src/agent_env/cli/task/create.py +++ b/src/agent_env/cli/task/create.py @@ -29,6 +29,7 @@ def create(filepath: Path, task_id: str, skip_validation: bool): """ from agent_env.bundle.parse import task_file_parts from agent_env.task import Task + from agent_env.task.lint import lint, unknown_fields click.echo(f"Reading steps from {filepath}...") with open(filepath) as f: @@ -39,13 +40,17 @@ def create(filepath: Path, task_id: str, skip_validation: bool): click.echo(f"Error: {e}", err=True) raise SystemExit(1) + if unknown := list(unknown_fields(steps)): + for finding in unknown: + click.echo(f"Error: {finding.text}", err=True) + raise SystemExit(1) built = _build_steps(steps) try: checked = Task(id=task_id, version=None, steps=[s for _, s in built], headline_verifiers=headlines) except ValueError as e: click.echo(f"Error: {e}", err=True) raise SystemExit(1) - _report_preflight(checked, skip_validation) + _report_preflight(checked, skip_validation, lint(checked, config=False)) task_steps = [] for i, (kwargs, step) in enumerate(built): @@ -90,8 +95,15 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]: return built -def _report_preflight(task, skip_validation: bool) -> None: - problems = task.preflight() +def _report_preflight(task, skip_validation: bool, findings) -> None: + """Refuse ``task`` for its lint errors and preflight problems, unless ``skip_validation``; print its lint warnings + either way. ``findings`` leave out ``config_problems``, which the preflight reports.""" + from agent_env.task.lint import Severity + + for finding in findings: + if finding.severity is Severity.WARNING: + click.echo(f"Warning: {finding.text}", err=True) + problems = [finding.text for finding in findings if finding.severity is Severity.ERROR] + task.preflight() if not problems: return click.echo(f"Preflight found {len(problems)} problem(s):", err=True) diff --git a/src/agent_env/cli/task/lint.py b/src/agent_env/cli/task/lint.py new file mode 100644 index 000000000..152e0a805 --- /dev/null +++ b/src/agent_env/cli/task/lint.py @@ -0,0 +1,112 @@ +"""Check a task for what would make it run or grade wrong, before any agent runs.""" + +import json +from pathlib import Path + +import click + + +@click.command() +@click.argument("path", required=False, type=click.Path(exists=True, path_type=Path)) +@click.option("--id", "task_id", default=None, help="A stored task's id, in place of PATH") +@click.option("--version", "task_version", default=None, type=int, help="Its version (defaults to latest)") +def lint(path: Path | None, task_id: str | None, task_version: int | None): + """Lint a task: a bundle folder's tasks, one task file, or a stored task (--id). + + \b + Errors are configs that can't run or grade correctly: an unknown field, a step's + own config problems, a headline that isn't a verifier, an aggregate source no + verifier writes. They also refuse agent-env run, bundle register, task create and + eval run. Warnings block nothing. Exits non-zero when there are errors. A bundle + folder is planned as a run plans it and --id reads the stored task, so both read + the configured stores; a task file on its own reads them only where a step's own + checks look up what it names, as load_artifact's do. + """ + if (path is None) == (task_id is None): + raise click.UsageError("give a bundle folder or task file, or --id, but not both") + if task_id is not None: + _report(*_stored(task_id, task_version)) + elif path.is_dir(): + _report(*_bundle(path)) + elif path.parent.name == "tasks" and path.suffix == ".json": + _report(*_bundle(path.parent.parent, path.stem)) + else: + _report(*_file(path)) + + +def _stored(task_id: str, task_version: int | None) -> tuple[list[str], list[str]]: + from agent_env.task import Task + from agent_env.task.lint import Severity, lint as lint_task + + task = Task.get(task_id, task_version) + findings = lint_task(task) + where = f"{task.id} v{task.version}" + return ([f"{where}: {f.text}" for f in findings if f.severity is Severity.ERROR], + [f"{where}: {f}" for f in findings if f.severity is Severity.WARNING]) + + +def _bundle(root: Path, task: str | None = None) -> tuple[list[str], list[str]]: + """A bundle's tasks as ``agent-env run`` plans them, so its references resolve as a run resolves them.""" + from agent_env.bundle import BundleError, BundleKind, parse_bundle + from agent_env.bundle.installed import installed_bundles + from agent_env.bundle.plan import plan_bundle + from agent_env.bundle.resolve import resolve_bundle + from agent_env.store.routing import namespace_routing + + with namespace_routing(): + try: + bundle = parse_bundle(root) + names = [task] if task else [entry.name for entry in bundle.entries if entry.kind is BundleKind.TASK] + plan = plan_bundle(resolve_bundle(bundle), tasks=names, installed=installed_bundles) + except BundleError as e: + return list(e.problems), [] + return [], [f"{task}: {finding}" for task, finding in plan.lint_findings] + + +def _file(path: Path) -> tuple[list[str], list[str]]: + """A task file on its own: its references name store ids, as ``task create`` reads them.""" + from agent_env.bundle.parse import task_file_parts + from agent_env.bundle.resolve import build_step + from agent_env.plugins import _registration + from agent_env.task import Task + from agent_env.task.lint import Severity, lint as lint_task, unknown_fields + from agent_env.task_step.registry import get_task_step_registry + + try: + steps, headlines = task_file_parts(json.loads(path.read_text())) + except (ValueError, json.JSONDecodeError) as e: + return [f"{path}: {e}"], [] + if unknown := list(unknown_fields(steps)): + return [f"{path}: {f.text}" for f in unknown], [] + registry, built, problems = get_task_step_registry(), [], [] + for step in steps: + step_type = step.get("type") + if not isinstance(step_type, str) or registry.get(step_type) is None: + note = _registration.failure_note(_registration.TASK_STEPS, step_type) if isinstance(step_type, str) else "" + problems.append(f"step {step.get('id')!r}: {step_type!r} is not a known step type{note}") + continue + try: + built.append(build_step(step)) + except Exception as e: + problems.append(f"step {step.get('id')!r}: {step_type} can't read it ({type(e).__name__}: {e})") + if problems: + return [f"{path}: {problem}" for problem in problems], [] + try: + task = Task(id=path.stem, version=None, steps=built, headline_verifiers=headlines) + except ValueError as e: + return [f"{path}: {e}"], [] + findings = lint_task(task, steps) + return ([f"{path}: {f.text}" for f in findings if f.severity is Severity.ERROR], + [f"{path}: {f}" for f in findings if f.severity is Severity.WARNING]) + + +def _report(errors: list[str], warnings: list[str]) -> None: + for line in warnings: + click.echo(line) + if errors: + click.echo(f"{len(errors)} error{'s' * (len(errors) != 1)}:", err=True) + for line in errors: + click.echo(f" - {line}", err=True) + raise SystemExit(1) + count = f"{len(warnings)} warning{'s' * (len(warnings) != 1)}" if warnings else "no problems" + click.echo(f"task lint: {count}") diff --git a/src/agent_env/cli/task/validate.py b/src/agent_env/cli/task/validate.py index 75ba10121..5737ec758 100644 --- a/src/agent_env/cli/task/validate.py +++ b/src/agent_env/cli/task/validate.py @@ -9,15 +9,20 @@ def validate(task_id: str, task_version: int | None): """Validate a stored task's step config without deploying anything. - Reports steps whose script artifact does not resolve, or resolves to the wrong - type, and deploy_env steps that couldn't deploy their env. Exits non-zero - when there are problems, so it can gate CI. + Reports what task lint finds, steps whose script artifact does not resolve, or + resolves to the wrong type, and deploy_env steps that couldn't deploy their env. + Exits non-zero when there are problems (lint's warnings aren't), so it can gate CI. """ from agent_env.task import Task + from agent_env.task.lint import Severity, lint from agent_env.task_step.task_step import TaskStep task = Task.get(task_id, task_version) - problems = task.preflight() + findings = lint(task, config=False) + for finding in findings: + if finding.severity is Severity.WARNING: + click.echo(f"{task.id} v{task.version}: {finding}", err=True) + problems = [finding.text for finding in findings if finding.severity is Severity.ERROR] + task.preflight() # Most step types have no preflight, so report coverage rather than implying a full check. checked = sum(1 for s in task.steps if type(s).preflight is not TaskStep.preflight) if problems: diff --git a/src/agent_env/config/describe.py b/src/agent_env/config/describe.py index 0cdc73163..8e88cea46 100644 --- a/src/agent_env/config/describe.py +++ b/src/agent_env/config/describe.py @@ -277,6 +277,7 @@ class FileSection: FileSection("model", ("model",), "config.runtime", dict.fromkeys(f.name for f in fields(ModelConfig))), FileSection("conversations", ("conversations",), "config.runtime", _resolved_keys("conversations")), FileSection("agents", ("agents",), "config.runtime", _resolved_keys("agents"), validated_agents_section), + FileSection("trajectories", ("trajectories",), "config.runtime", _resolved_keys("trajectories")), FileSection("sandbox", ("sandbox",), "providers.sandbox_providers.sandbox_provider", {"default": None, "agent_default": None, "attribution": None, "providers": Named(_SEAM)}), FileSection("state", ("state",), "providers.env_state.env_state_provider", {"providers": Named(_SEAM)}), diff --git a/src/agent_env/config/model_call.py b/src/agent_env/config/model_call.py new file mode 100644 index 000000000..10c0776be --- /dev/null +++ b/src/agent_env/config/model_call.py @@ -0,0 +1,131 @@ +"""The one place core calls a model itself, as its judges do: the route a call takes, the key it carries, and what it +called and what it cost. Nothing else in core imports LiteLLM (``tst/unit/config/test_one_model_call_path.py``), and +this module imports it only where it's used, since its import takes seconds.""" + +from __future__ import annotations + +import contextlib +import io +import math +from dataclasses import dataclass, replace +from typing import Any, Callable, Optional + +from agentenv_protocol.a2a_agent import Usage + +from agent_env.config.errors import ConfigError +from agent_env.config.model import ModelCallConfig, ModelParam, api_root, endpoint_host, no_key_for +from agent_env.config.model_pricing import Cost, ReportedBy, Tokens, litellm_prices, price +from agent_env.config.runtime import get_config + + +@dataclass(frozen=True) +class ModelRoute: + """How a call to ``model`` goes: the name and base LiteLLM is handed (``litellm_model``, ``call.api_base``) and + the key and params it carries. ``model`` is what the call records.""" + + model: str + litellm_model: str + call: ModelCallConfig + + @property + def endpoint_host(self) -> Optional[str]: + return endpoint_host(self.call.api_base) + + +@dataclass(frozen=True) +class ModelReply: + """A call's answer, the model it called and the host it reached, its usage as the response carried it, and its + cost: what the endpoint billed when it says, else LiteLLM's price map's figure, else unpriced.""" + + content: str + model: str + endpoint_host: Optional[str] + usage: Optional[dict] + cost: Cost + + def fields(self) -> dict[str, Any]: + """The record of the call, beside its answer: ``usage``, ``model``, ``endpoint_host`` and the cost's keys.""" + return {"usage": self.usage, "model": self.model, "endpoint_host": self.endpoint_host, **self.cost.fields()} + + +def resolve_route(model: str, *, api_key: Optional[str], api_base: Optional[str], who: str, + key_how: str) -> ModelRoute: + """The route a call to ``model`` takes, given the key and base its caller was handed (either may be None, a key an + ``env:`` or ``secret:`` reference). A base other than the configured endpoint needs a key of its own, since the + configured one isn't sent there: without one this refuses, as ``who``, saying ``key_how`` to give one.""" + import litellm + + config = get_config() + key = config.resolve_reference(api_key) + if api_base and not key and not config.is_model_endpoint(api_base): + raise ConfigError(f"{who}: {no_key_for(api_base, key_how)}") + call = config.resolve_model_call(model, default_api_key=key, base_override=api_base) + name, base = litellm_route(model, call.api_base, litellm.get_llm_provider, call.params.get("custom_llm_provider")) + return ModelRoute(model, name, replace(call, api_base=base)) + + +def litellm_route(model: str, api_base: Optional[str], provider_of: Callable[..., Any], + custom_provider: Optional[str] = None) -> tuple[str, Optional[str]]: + """The model name and base a call hands LiteLLM. Through an endpoint, a bare name goes by what LiteLLM makes of + it: an Anthropic one keeps that route, with the base's trailing ``/v1`` dropped since its client appends + ``/v1/messages``; an OpenAI one goes as configured; any other, which LiteLLM would send to that provider's own API + (``gemini-*`` to Vertex), or one it can't place, goes through the endpoint as ``openai/``. A provider prefix + in the name, or a ``custom_llm_provider`` among the call's params, keeps the route it names. ``provider_of`` is + LiteLLM's ``get_llm_provider``, which prints a banner before refusing a name it can't place.""" + if api_base is None: + return model, None + try: + with contextlib.redirect_stdout(io.StringIO()): + provider = provider_of(model, custom_llm_provider=custom_provider)[1] + except Exception: + return f"openai/{model}", api_base + if provider == "anthropic": + return model, api_root(api_base) + if provider == "openai" or custom_provider or "/" in model: + return model, api_base + return f"openai/{model}", api_base + + +async def complete(route: ModelRoute, messages: list[dict], *, timeout: Optional[float], + response_format: Optional[dict] = None) -> ModelReply: + """One chat completion by ``route``, raising what LiteLLM raises.""" + import litellm + + request: dict[str, Any] = dict(model=route.litellm_model, messages=messages, timeout=timeout, + **route.call.client_kwargs()) + if response_format is not None: + request[ModelParam.RESPONSE_FORMAT] = response_format + response = await litellm.acompletion(**request) + usage = getattr(response, "usage", None) + usage = usage.model_dump() if hasattr(usage, "model_dump") else usage + usage = usage if isinstance(usage, dict) else None + tokens = Tokens.of(Usage.from_model_response(usage).to_dict()) if usage else None + cost = price(tokens, _billed(getattr(response, "_hidden_params", None)), model=route.model, + prices=litellm_prices(), reported_by=ReportedBy.PROVIDER) + return ModelReply(response.choices[0].message.content or "", route.model, route.endpoint_host, usage, cost) + + +def setup_errors() -> tuple[type[BaseException], ...]: + """What a call raises when it can't work as set up, as opposed to a flaky failure: agent-env's ConfigError (no key, + missing config) and LiteLLM's AuthenticationError, PermissionDeniedError, NotFoundError (an unknown model), + BadRequestError (with its kinds: an unknown provider, unsupported params, a prompt over the context window), + UnprocessableEntityError and BudgetExceededError.""" + import litellm + + return (ConfigError, litellm.AuthenticationError, litellm.PermissionDeniedError, litellm.NotFoundError, + litellm.BadRequestError, litellm.UnprocessableEntityError, litellm.BudgetExceededError) + + +def _billed(hidden: object) -> Optional[float]: + """What the endpoint billed for the call: a LiteLLM proxy's ``x-litellm-response-cost`` header, as LiteLLM reads + it, never LiteLLM's own estimate, which its ``response_cost`` holds when the header is missing. None without the + header, or for one that isn't a cost.""" + from litellm.cost_calculator import get_response_cost_from_hidden_params + + if not isinstance(hidden, dict): + return None + try: + billed = get_response_cost_from_hidden_params(hidden) + except (TypeError, ValueError): + return None + return billed if billed is not None and math.isfinite(billed) and billed >= 0 else None diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index c6e49565e..d632dde39 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -103,6 +103,15 @@ def _validated_a2a_agent_id(value: object, source: str) -> str: return value +_TRAJECTORIES_CONFIG_SECTION = "trajectories" + + +def _validated_trajectory_formats(value: object, source: str) -> tuple[str, ...]: + if not isinstance(value, list) or not all(isinstance(format, str) and format.strip() for format in value): + raise ConfigError(f"{source} must be a list of trajectory format names, such as [\"atif/1\"], got {value!r}") + return tuple(value) + + def _validated_env_id(value: object, source: str) -> str: if not isinstance(value, str) or not value.strip(): raise ConfigError(f"{source} must be a non-empty string env id, got {value!r}") @@ -163,6 +172,8 @@ def file_label(self) -> str: *(ResolvedKey((_ENVS_CONFIG_SECTION, name), field=name, default=env_id, default_label="built-in default in config.runtime") for name, env_id in _DEFAULT_ENV_IDS.items()), + ResolvedKey((_TRAJECTORIES_CONFIG_SECTION, "formats"), default=(), + default_label="built-in default in config.runtime: none, native only"), ) _KEY_BY_NAME = {key.name: key for key in RESOLVED_KEYS} @@ -426,6 +437,18 @@ def get_default_human_a2a_url(self) -> str: return _validated_human_a2a_url(resolved, key.file_label) raise ConfigError(f"No human-A2A URL configured: set [{_HUMAN_A2A_CONFIG_SECTION}] {_HUMAN_A2A_CONFIG_KEY} or AGENT_ENV_HUMAN_A2A_URL") + def get_trajectory_formats(self) -> tuple[str, ...]: + """The trajectory formats beside native a run prescribes to each agent whose trajectory it reads: + ``[trajectories] formats`` in config.toml, else none, so agents build native only.""" + key = _KEY_BY_NAME[f"{_TRAJECTORIES_CONFIG_SECTION}.formats"] + for layer in self.key_candidates(key): + if layer.kind == KIND_DEFAULT: + return tuple(layer.raw) + self.section(_TRAJECTORIES_CONFIG_SECTION) + resolved = config_loader.interpolate(layer.raw, secret_resolver=self._resolve_secret) + return _validated_trajectory_formats(resolved, key.file_label) + return () + def get_default_env_id(self, name: str) -> str: """The default infra env id ``name`` (``default_gateway_env_id``, ``default_service_db_env_id`` or ``default_website_browser_env_id``): one set in code > ``[envs] `` in config.toml > the built-in id.""" diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index baea4a7f6..5ffd59d7d 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -16,7 +16,8 @@ alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/ `envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `ubuntu-24.04-browser`: its screen is the computer's one website browser's active tab, its `screen` feature `page`. A screenshot is the page, and input reaches it as CDP input events, both through `browser/v1`'s `page/*` operations on the browser's own server; -a tab that paints nothing shows white. It serves the page group too: `computer_navigate`, `computer_history`, +a tab that paints nothing shows white. Its `screenshot_delay_ms`, how long after an input its screenshot is +taken, is shorter than the desktop drivers', since the page shows a key's scroll sooner. It serves the page group too: `computer_navigate`, `computer_history`, `computer_tabs`, `computer_dialog` and `computer_upload`, and every frame names the page, its tab and what is open on it. The browser ending on its own fails the computer for good: its ready check is `final`, and its tools answer with why. Each call it makes to the browser for an agent's tool carries that call, its tool, id, arguments and the caller's role, in the `AgentEnv-Tool-Call` and `AgentEnv-Role` headers, so the browser's log credits the action to it. Each `browser_*` call is an entry too, begun and ended with `calls/begin` and `calls/end`, which Playwright MCP's CDP commands meanwhile, and the tab events they cause, name as `for`. Name it as `page`, or by its id, `@local/agentenv-framework/drivers/page`. diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index b3d25a5b4..06f9650a2 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -102,6 +102,8 @@ class PageDriver(ComputerDriverEnvironment): keys = sorted({*NAMED, *string.ascii_letters, *string.digits, *PUNCTUATION, *SHIFTED}) recording = Recording(max_fps=10) screen_kind = "page" + #: Shorter than a desktop driver's: the page shows a key's scroll sooner than a desktop's browser window does. + screenshot_delay_ms = 300 notes = ("This computer is one web page: the browser's active tab, with no address bar, tab strip or desktop " "around it. Its screen is the page's viewport. Go to a URL with computer_navigate, back, forward or " "reload with computer_history, and list, open, switch or close tabs with computer_tabs. Each screenshot " diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 1f5e1ae87..193914bba 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -175,8 +175,8 @@ async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> boo return False else: try: - await legacy_protocol.reset_via_rest(legacy_protocol.environment_base_url(env._gateway_url, name, mcp=mcp), - timeout=60) + legacy_url = legacy_protocol.environment_base_url(env._gateway_url, name, mcp=mcp) + await legacy_protocol.reset_via_rest(legacy_url, timeout=60, headers=data_plane_headers(env, legacy_url)) except httpx.HTTPStatusError as e: if e.response.status_code != 404: raise diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index c26a15de0..a42867b16 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -294,9 +294,9 @@ async def _exec_in_server(self, *command: str) -> None: async def _load_environment_artifact_legacy(self, file_artifact) -> None: container_path = await self._copy_artifact_into_container(file_artifact, file_artifact.filename) - await legacy_protocol.reset( - self._gateway_url, self.environment_name, container_path, max_retries=self._MCP_MAX_RETRIES - ) + base_url = legacy_protocol.environment_base_url(self._gateway_url, self.environment_name, mcp=True) + await legacy_protocol.reset(self._gateway_url, self.environment_name, container_path, + max_retries=self._MCP_MAX_RETRIES, headers=data_plane_headers(self, base_url)) async def validate(self, on_progress: Optional[Callable[[str], None]] = None) -> str: """Run the env validator task and return the task instance ID. diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index 0875528a8..59790d0c2 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -372,9 +372,10 @@ async def _load_environment_artifact_legacy(self, file_artifact) -> None: container_path = await self._copy_artifact_into_container(file_artifact, file_artifact.filename) base_url = legacy_protocol.environment_base_url(self._gateway_url, self.environment_name, mcp=False) from agent_env.env.gateway.constants import DATA_PLANE_LOAD_TIMEOUT_S - await legacy_protocol.reset_via_rest(base_url, timeout=60) + headers = data_plane_headers(self, base_url) + await legacy_protocol.reset_via_rest(base_url, timeout=60, headers=headers) try: - await legacy_protocol.add_via_rest(base_url, container_path, timeout=DATA_PLANE_LOAD_TIMEOUT_S) + await legacy_protocol.add_via_rest(base_url, container_path, timeout=DATA_PLANE_LOAD_TIMEOUT_S, headers=headers) except Exception as e: logger.error( "load_environment_artifact: /api/add failed after /api/reset succeeded; " diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 35e4d6957..8c6e152c1 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -198,15 +198,26 @@ def sources(names: list[str]) -> str: TOOL_DISABLE_ACTION = "disable" TOOL_ENABLE_ACTION = "enable" # The harness's routes that take the deploy's control token: the ones that change tool access, triggers and -# notifications. The gateway also takes it for the data plane and for an MCP server's own endpoint behind its proxy -# (Gateway._takes_control_token). Every other route stays open for now, the clock's writes included: callers outside -# agent-env set the clock over plain HTTP and don't send the token yet. +# notifications. The gateway also takes it, behind its proxy, for the data plane, for an MCP server's own endpoint and +# for the LEGACY_STATE_ROUTES (Gateway._takes_control_token). Every other route stays open for now, the clock's writes +# included: callers outside agent-env set the clock over plain HTTP and don't send the token yet. CONTROL_ROUTES = frozenset({ f"/tools/{TOOL_DISABLE_ACTION}", f"/tools/{TOOL_ENABLE_ACTION}", "/triggers/register", "/triggers/remove", "/triggers/clear", "/notifications/config", "/notifications/take", "/notifications/clear", CLOCK_FOLLOWERS_PATH, }) +# The older REST routes a backing server serves beside its data plane, which reset, load or export its state. +# Behind the proxy, /svc// takes the control token for each, and anything under it, on a gateway that has one. +# A site's /website route serves none of them, since a site with no frontend of its own serves its backend there and +# an older frontend passes them on. +LEGACY_STATE_ROUTES = ( + "/api/reset", "/api/add", "/api/bridge/apply", + "/export-state", "/export-snapshot", "/snapshot-dirty", + "/reset-memory", "/validate-rows", + "/api/_state", "/api/_reset", "/api/debug/seed", +) +SITE_LEGACY_STATE_ROUTES = frozenset(LEGACY_STATE_ROUTES) class GatewayMode(str, Enum): diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index ae2af8ed6..c3045ad1e 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -74,6 +74,8 @@ WILDCARD, PGWEB_CONSOLE, GatewayMode, + LEGACY_STATE_ROUTES, + SITE_LEGACY_STATE_ROUTES, tool_name_clashes_text, ) from .browser_proxy import PING_INTERVAL_S, BrowserRoutes, TokenRedaction @@ -692,9 +694,9 @@ def _is_pgweb(scope: dict) -> bool: def _takes_control_token(self, scope: dict) -> bool: """Whether a request takes the deploy's control token: a control route, the env's data plane, any path of a SQL - console behind /svc, or one of ``token_paths`` behind /svc, by the path as routed or by the one the backing - server receives (``reaches``), after dot-segment normalisation. A site's /website route never reaches its data - plane.""" + console behind /svc, or one of ``token_paths`` behind /svc (a backing server's data plane, its MCP endpoint and + its LEGACY_STATE_ROUTES), by the path as routed or by the one the backing server receives (``reaches``), after + dot-segment normalisation. A site's /website route never reaches its data plane.""" path = scope["path"] if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): return True @@ -970,12 +972,13 @@ async def _proxy_website_request(self, request: Request) -> Response: """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers. A site's data plane and card, and every path its card advertises for agentenv (``site_reserved_paths``), are never reached through its route, so an agent browsing the site can't read its data or what grades it; agentenv - reaches them through /svc//.""" + reaches them through /svc//. On a gateway with a token, neither is SITE_LEGACY_STATE_ROUTES + (``_site_reserved``).""" app = request.path_params["service_name"] card = await self._site_card(app) if card is None: return self._json_response({"error": "website card unavailable"}, 503) - if reserved_site_request(request, card.reserved): + if reserved_site_request(request, self._site_reserved(card), self.website_urls.get(app, "http://site")): return self._json_response({"error": "not found"}, 404) response = await self._proxy_rest_request(request, url_map=self.website_urls, site_names=self._names(app, card)) engine = self._trigger_engine @@ -993,14 +996,15 @@ async def _proxy_website_request(self, request: Request) -> Response: async def _proxy_website_socket(self, websocket: WebSocket) -> None: """Relay a WebSocket on a site's route to its server, as its HTTP requests are: its data plane and card, and - every path its card advertises for agentenv (``site_reserved_paths``), are refused, and it goes upstream with - the role, role key and forwarded address an HTTP request would carry.""" + every path its card advertises for agentenv (``site_reserved_paths``), are refused, as ``_site_reserved`` + says, and it goes upstream with the role, role key and forwarded address an HTTP request would carry.""" service_name = websocket.path_params["service_name"] card = await self._site_card(service_name) if card is None: await refuse_websocket(websocket, self._json_response({"error": "website card unavailable"}, 503)) return - if reserved_site_request(websocket, card.reserved): + if reserved_site_request(websocket, self._site_reserved(card), + self.website_urls.get(service_name, "http://site")): await refuse_websocket(websocket, self._json_response({"error": "not found"}, 404)) return base_url = self.website_urls.get(service_name) @@ -1011,6 +1015,11 @@ async def _proxy_website_socket(self, websocket: WebSocket) -> None: open_timeout=self.REST_PROXY_CONNECT_TIMEOUT_S, withheld=self._withheld, site_names=self._names(service_name, card)) + def _site_reserved(self, card: SiteCard) -> frozenset[str]: + """The paths a site's route never reaches: those its card advertises for agentenv, and on a gateway with a + token SITE_LEGACY_STATE_ROUTES, which an older site's frontend passes on to its backend.""" + return card.reserved | SITE_LEGACY_STATE_ROUTES if self._control_bearer else card.reserved + async def _fire_website_triggers(self, request: Request, app: str, status: int) -> tuple[list[asyncio.Task], float | None]: """Feed an answered website request to the trigger engine (fail-open).""" @@ -1942,10 +1951,11 @@ def mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, str] def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: """The paths under /svc// that take the gateway's control token, by key: every backing server's data plane, - extensions included, and an MCP server's own endpoint (``mcp_endpoints``), which skips the gateway's trajectory and - triggers. A child env registered under its own name, a computer, takes its own token, so none of its paths are here.""" + extensions included, its LEGACY_STATE_ROUTES, and an MCP server's own endpoint (``mcp_endpoints``), which skips + the gateway's trajectory and triggers. A child env registered under its own name, a computer, takes its own token, + so none of its paths are here.""" endpoints = mcp_endpoints(servers, website_urls, rest_proxy_urls) - return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ())) + return {key: (RPC_PATH, *LEGACY_STATE_ROUTES, *((endpoints[key],) if key in endpoints else ())) for key, base in rest_proxy_urls.items() if key in website_urls or route_environment_name(key, base, servers, website=False) != key or not any(server.name == key for server in servers)} @@ -2012,23 +2022,29 @@ def site_reserved_paths(card: dict, base: str = "") -> frozenset[str]: return frozenset(p for p in paths if p != "/") -def site_request_path(request: HTTPConnection) -> str | None: - """The path a site's server receives for a request on its route: the raw path forwarded, its dot segments removed - by httpx and then decoded. It differs from the path as routed here when an encoded slash sits beside a dot segment - (``public%2Fx/../api`` reaches ``/api``). None for a request-target the proxy can't build a URL for: ``forward()`` - refuses it.""" +def site_request_path(request: HTTPConnection, base_url: str = "http://site") -> str | None: + """The path a site's server receives for a request on its route, relative to the path of its address + ``base_url``: the raw path forwarded, decoded and its dot segments removed, with the address's path taken off. It differs from the path as routed here when an encoded slash sits beside a dot segment + (``public%2Fx/../api`` reaches ``/api``), or when dot segments climb out of the address's path and back into it + (``../base/api`` under ``/base`` reaches ``/api``). A path that climbs out and stays out is given whole. None for + a request-target the proxy can't build a URL for: ``forward()`` refuses it.""" try: - return httpx.URL(upstream_url("http://site", request)).path + sent = _normalized(httpx.URL(upstream_url(base_url.rstrip("/"), request)).path) + under = _normalized(httpx.URL(base_url).path).rstrip("/") except (ValueError, httpx.InvalidURL): return None + if under and (sent == under or sent.startswith(under + "/")): + return sent[len(under):] or "/" + return sent -def reserved_site_request(request: HTTPConnection, reserved: frozenset[str] = frozenset()) -> bool: +def reserved_site_request(request: HTTPConnection, reserved: frozenset[str] = frozenset(), + base_url: str = "http://site") -> bool: """Whether a request or WebSocket on a site's route reaches its data plane or env card, or one of the paths - ``reserved`` names (``site_reserved_paths``): by the path as routed here, or by the one its server receives - (``site_request_path``). A request-target the proxy can't build a URL for isn't reserved here: ``forward()`` - refuses it.""" + ``reserved`` names (``site_reserved_paths``): by the path as routed here, or by the one its server at + ``base_url`` receives (``site_request_path``). A request-target the proxy can't build a URL for isn't reserved + here: ``forward()`` refuses it.""" if reserved_site_path("/" + request.path_params.get("path", ""), reserved): return True - sent = site_request_path(request) + sent = site_request_path(request, base_url) return sent is not None and reserved_site_path(sent, reserved) diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 18d78e234..216397913 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -209,21 +209,28 @@ async def reset_via_rest( mock_data_path: Optional[str] = None, timeout: int = 30, verify: bool = True, + *, + headers: Optional[dict[str, str]] = None, ) -> dict: - """POST ``{base_url}/api/reset``. Sends the seed path when given. Raises on non-2xx.""" + """POST ``{base_url}/api/reset`` with ``headers``, else the run's grant (``request_headers``). Sends the seed path + when given. Raises on non-2xx.""" body = {"mock_data_path": mock_data_path} if mock_data_path is not None else None + url = f"{base_url}/api/reset" async with httpx.AsyncClient(verify=verify) as client: - response = await client.post(f"{base_url}/api/reset", json=body, timeout=timeout) + response = await client.post(url, json=body, timeout=timeout, headers=protocol_v1.request_headers(url, headers)) response.raise_for_status() result = response.json() logger.info(f"load_environment_artifact: REST reset succeeded: {result}") return result -async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True) -> dict: - """POST ``{base_url}/api/add`` with a backend file path. Raises on non-2xx.""" +async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify: bool = True, *, + headers: Optional[dict[str, str]] = None) -> dict: + """POST ``{base_url}/api/add`` with a backend file path and ``headers``, else the run's grant. Raises on non-2xx.""" + url = f"{base_url}/api/add" async with httpx.AsyncClient(verify=verify) as client: - response = await client.post(f"{base_url}/api/add", json={"file_path": file_path}, timeout=timeout) + response = await client.post(url, json={"file_path": file_path}, timeout=timeout, + headers=protocol_v1.request_headers(url, headers)) response.raise_for_status() result = response.json() logger.info(f"load_environment_artifact: add response: {result}") @@ -231,36 +238,41 @@ async def add_via_rest(base_url: str, file_path: str, timeout: int = 120, verify async def export_state( - gateway_url: str, environment_name: str, timeout: int = EXPORT_STATE_TIMEOUT_SECONDS, verify: bool = True + gateway_url: str, environment_name: str, timeout: int = EXPORT_STATE_TIMEOUT_SECONDS, verify: bool = True, *, + headers: Optional[dict[str, str]] = None, ) -> dict: - """GET ``/svc/mcp-{name}/export-state`` — the legacy state snapshot for one service.""" - return await _export_state_at(environment_base_url(gateway_url, environment_name, mcp=True), timeout, verify) + """GET ``/svc/mcp-{name}/export-state``, the legacy state snapshot for one service, with ``headers``, else the + run's grant.""" + return await _export_state_at(environment_base_url(gateway_url, environment_name, mcp=True), timeout, verify, headers) -async def _export_state_at(base_url: str, timeout: int, verify: bool) -> dict: +async def _export_state_at(base_url: str, timeout: int, verify: bool, headers: Optional[dict[str, str]]) -> dict: + url = f"{base_url}/export-state" async with httpx.AsyncClient(verify=verify) as client: - response = await client.get(f"{base_url}/export-state", timeout=timeout) + response = await client.get(url, timeout=timeout, headers=protocol_v1.request_headers(url, headers)) response.raise_for_status() return response.json() async def service_state(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str) -> dict: - """An MCP service's state as JSON: its v1 ``data/get`` answer when that is data, asked with the deployment's - control token, else ``GET /export-state``. + """An MCP service's state as JSON: its v1 ``data/get`` answer when that is data, else ``GET /export-state``, each + asked with the deployment's control token. A service that exports a file from ``data/get`` (a bundle of its database, to reload from) still serves its state as JSON at ``/export-state``. """ base_url = await v1_base_url(deployed, gateway_url, environment_name, mcp=True) if base_url is None: - return await export_state(gateway_url, environment_name) - response = await protocol_v1.get_data(base_url, - headers=deployed.control_headers_for(base_url) if deployed is not None else None) + legacy_url = environment_base_url(gateway_url, environment_name, mcp=True) + return await export_state(gateway_url, environment_name, + headers=(deployed.control_headers_for(legacy_url) or None) if deployed is not None else None) + headers = (deployed.control_headers_for(base_url) or None) if deployed is not None else None + response = await protocol_v1.get_data(base_url, headers=headers) if not response.parts: return {} if isinstance(response.parts[0], DataPart): return response.parts[0].data - return await _export_state_at(base_url, timeout=EXPORT_STATE_TIMEOUT_SECONDS, verify=True) + return await _export_state_at(base_url, timeout=EXPORT_STATE_TIMEOUT_SECONDS, verify=True, headers=headers) async def offers_tool(mcp_url: str, name: str) -> bool: @@ -322,12 +334,17 @@ async def reset( environment_name: str, mock_data_path: str, max_retries: int = DEFAULT_MCP_MAX_RETRIES, + *, + headers: Optional[dict[str, str]] = None, ) -> Optional[dict]: - """Reset an MCP service to a seed: try REST ``/api/reset``, fall back to the MCP tool.""" + """Reset an MCP service to a seed: try REST ``/api/reset`` with ``headers``, fall back to the MCP tool. A 401 or + 403 raises: it means the token was dropped or wrong, and the tool would reset the service without one.""" base_url = environment_base_url(gateway_url, environment_name, mcp=True) try: - return await reset_via_rest(base_url, mock_data_path) + return await reset_via_rest(base_url, mock_data_path, headers=headers) except Exception as rest_err: + if isinstance(rest_err, httpx.HTTPStatusError) and rest_err.response.status_code in (401, 403): + raise # TODO remove MCP tool fallback once all usages of MCP servers are on REST logger.info(f"load_environment_artifact: REST reset failed ({type(rest_err).__name__}), falling back to MCP tool") await reset_via_mcp_tool(f"{gateway_url}/mcp", environment_name, mock_data_path, max_retries=max_retries) diff --git a/src/agent_env/openenv/app.py b/src/agent_env/openenv/app.py index 869b64abf..bc7190c5e 100644 --- a/src/agent_env/openenv/app.py +++ b/src/agent_env/openenv/app.py @@ -37,6 +37,7 @@ def build_app( sandbox: str | None = None, max_concurrent_envs: int = 8, token: str | None = None, + skip_lint: bool = False, ) -> FastAPI: """The OpenEnv server for ``bundle``'s tasks: a folder, an installed bundle's name or an ``hf://`` reference, as ``agent-env run BUNDLE`` reads it, served as the split ``all`` and a split per eval; or a mapping of split name to @@ -45,14 +46,15 @@ def build_app( ``reward`` makes the reward of a task's scores: a step id, or a function of the scores by step id; without it a task that records one score has that one. Each bundle is prepared as ``agent-env run`` prepares it before its - first task, without the agent a client plays, so every task must pass its checks; one that can't run as an - episode is then left out and logged. Namespace routing is on from the app's startup to its shutdown, which tears - down every open episode. With ``token``, every request and WebSocket but ``/health`` must carry + first task, without the agent a client plays, so every task must pass its checks, but for task lint's own rules + with ``skip_lint``, as ``agent-env run --skip-lint``; one that can't run as an episode is then left out and + logged. Namespace routing is on from the app's startup to its shutdown, which tears down every open episode. + With ``token``, every request and WebSocket but ``/health`` must carry ``Authorization: Bearer ``, for a server others can reach: without it, anyone who reaches the server deploys and drives its envs. Raises BundleError when a bundle can't be prepared or serves no task, and ValueError when ``reward`` can't score every task.""" with namespace_routing(): - prepared = {split: _prepared(source, sandbox) for split, source in _sources(bundle).items()} + prepared = {split: _prepared(source, sandbox, skip_lint) for split, source in _sources(bundle).items()} served = Served(splits=_served(_splits(bundle, prepared), reward), sandbox=sandbox) return _app(served, _env_name(prepared), max_concurrent_envs, token) @@ -102,14 +104,14 @@ def _sources(bundle: _Source | Mapping[str, _Source]) -> dict[str | None, _Sourc return dict(bundle) if isinstance(bundle, Mapping) else {None: bundle} -def _prepared(source: _Source, sandbox: str | None) -> PreparedBundle: +def _prepared(source: _Source, sandbox: str | None, skip_lint: bool = False) -> PreparedBundle: """``source`` prepared with every task and eval selected, so ``all`` holds every task, evals or not.""" root, id_root = (source, None) if isinstance(source, Path) else locate_bundle(source)[:2] entries = parse_bundle(Path(root), id_root=id_root).entries return prepare_bundle( root, tasks=[entry.name for entry in entries if entry.kind is BundleKind.TASK], evals=[entry.name for entry in entries if entry.kind is BundleKind.EVAL], sandbox=sandbox, - on_progress=logger.info, id_root=id_root, played=True) + on_progress=logger.info, id_root=id_root, played=True, skip_lint=skip_lint) def _splits(bundle: _Source | Mapping[str, _Source], diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index f8b20301e..051cf62ba 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -22,6 +22,7 @@ import shutil import signal import socket +import stat import subprocess import tempfile from ipaddress import ip_address @@ -126,9 +127,10 @@ def trust_env(directory: str) -> dict[str, str]: # up on Linux, the VM Docker Desktop or Rancher Desktop runs Docker in included; 49152 up on macOS and Windows) and # below Kubernetes' node ports (30000-32767). _HOST_PORTS = range(20000, 30000) -# Holds the last host port handed out, in the per-user state root's locks, so processes sharing that root hand them -# out in turn. +# Holds the last host port handed out, in a folder of the user's own in this machine's /tmp, so every process the user +# runs here hands them out in turn, whatever its state root or TMPDIR. _HOST_PORT_CURSOR = "host-ports" +_MACHINE_TMP = Path("/tmp") def _local_sandbox_root_path() -> Path: @@ -155,13 +157,11 @@ def _local_sandbox_root() -> Path: def _free_host_port() -> int: """The next port in ``_HOST_PORTS`` after the last one handed out, skipping those in use. - Docker binds it later, so nothing may take it meanwhile: a deploy sharing this state root draws it again only after + Docker binds it later, so nothing may take it meanwhile: another deploy on this machine draws it again only after the whole range has gone round, and outgoing connections don't use the range by default. The probe alone could not keep it: it releases the port, and can't see the sockets of a VM Docker runs in. """ - locks = state_root() / "locks" - ensure_state_dir(locks) - fd = os.open(locks / _HOST_PORT_CURSOR, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + fd = os.open(_host_port_cursor(), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) try: fcntl.flock(fd, fcntl.LOCK_EX) last = os.read(fd, 16).strip() @@ -177,6 +177,21 @@ def _free_host_port() -> int: raise RuntimeError(f"Every host port from {_HOST_PORTS.start} to {_HOST_PORTS.stop - 1} is in use") +def _host_port_cursor() -> Path: + """The cursor's file: in this user's own folder of the machine's /tmp, not of ``TMPDIR``, which a process may set + for itself, so processes whose state roots or temp folders differ, as each xdist worker's state root does, share + it; in the state root's locks when that folder can't be made or isn't this user's alone.""" + folder = _MACHINE_TMP / f"agentenv-{os.getuid()}" + with contextlib.suppress(OSError): + folder.mkdir(mode=0o700) + held = folder.lstat() if folder.exists() else None + if held and stat.S_ISDIR(held.st_mode) and held.st_uid == os.getuid() and not held.st_mode & 0o077: + return folder / _HOST_PORT_CURSOR + locks = state_root() / "locks" + ensure_state_dir(locks) + return locks / _HOST_PORT_CURSOR + + def _bindable(port: int) -> bool: """Whether ``port`` is free on every interface, as docker binds it when publishing: a port free on loopback alone can still be taken on another interface. A bind that fails for another reason raises.""" diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index 0a519869d..96b4ee7d0 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -271,6 +271,11 @@ def local(self) -> LocalNamespaceDocumentStore: return self.namespaces[LOCAL_NAMESPACE] def find_one(self, collection: str, filter: Filter, sort: Optional[Sort] = None) -> Optional[dict]: + if not sort or not sort.keys: + for store in self._readers(collection, filter): + if (doc := self._read(store, lambda: store.find_one(collection, filter, sort))) is not None: + return doc + return None found = [ doc for store in self._readers(collection, filter) if (doc := self._read(store, lambda: store.find_one(collection, filter, sort))) is not None diff --git a/src/agent_env/task/lint.py b/src/agent_env/task/lint.py new file mode 100644 index 000000000..26a2483d7 --- /dev/null +++ b/src/agent_env/task/lint.py @@ -0,0 +1,280 @@ +"""What a task gets wrong before any agent runs (``agent-env task lint``). An error is a config that can't run or grade +correctly, and refuses the run; everything else is a warning, which blocks nothing.""" + +from __future__ import annotations + +import collections +import copy +import difflib +from dataclasses import dataclass +from enum import Enum +from typing import Any, Iterable, Iterator, Mapping, Sequence + +from agent_env.entity.refs import RefRole, ref_sites +from agent_env.store.ids import is_local_id +from agent_env.task.task import Task +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_step import StepRole, TaskStep, attach_base_fields +from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, trajectory_mistakes_key +from agent_env.task_step.task_steps.verifiers.verifier import VerifierTaskStep + + +class Severity(str, Enum): + ERROR = "error" + WARNING = "warning" + + +@dataclass(frozen=True) +class Finding: + severity: Severity + rule: str + message: str + step_id: str | None = None + about: str | None = None # what the finding names, for a summary to count by: an unpinned reference's entity kind + + @property + def text(self) -> str: + """The message with the step it's about, as a refusal lists it.""" + return f"step {self.step_id!r}: {self.message}" if self.step_id is not None else self.message + + def __str__(self) -> str: + return f"{self.severity.value}: {self.text}" + + +def lint(task: Task, step_dicts: Sequence[Mapping[str, Any]] | None = None, *, config: bool = True, + unpinned: bool = True) -> list[Finding]: + """``task``'s findings, errors first. ``step_dicts``, the task's steps as a file holds them, adds the unknown-field + rule, which a stored task can't break since storing drops unknown fields. ``config`` adds each step's own + ``config_problems`` as errors; a caller that runs ``Task.preflight`` already has them. ``unpinned`` warns for a store + entity read with no version; a bundle's plan leaves it out, since a plan records each version it reads and a + registration pins it. Reads no store itself; a step's ``config_problems`` may, as load_artifact's do.""" + findings = [ + *(unknown_fields(step_dicts) if step_dicts is not None else ()), + *(_step_configs(task.steps) if config else ()), + *_step_warnings(task.steps), + *_headlines(task), + *_aggregate_sources(task.steps), + *(_unpinned(task.steps) if unpinned else ()), + *_coverage(task), + ] + return sorted(findings, key=lambda finding: finding.severity is not Severity.ERROR) + + +#: How many of the tasks a finding is in a run names, before how many more. +SHOWN_TASKS = 3 +UNPINNED = "unpinned-reference" + + +def said(findings: Iterable[tuple[str, Finding]]) -> list[str]: + """``findings``, each with the task it's in, as a run says them: each kind once, errors first, after its task when + only one has it, else with how many and the first few, so a bundle of a thousand tasks says each kind once.""" + where: dict[tuple[Severity, str], list[str]] = {} + for task, finding in findings: + where.setdefault((finding.severity, finding.text), []).append(task) + lines = [] + for (severity, text), tasks in sorted(where.items(), key=lambda item: item[0][0] is not Severity.ERROR): + if len(tasks) == 1: + lines.append(f"{tasks[0]}: {severity.value}: {text}") + else: + more = len(tasks) - SHOWN_TASKS + shown = ", ".join(tasks[:SHOWN_TASKS]) + (f" and {more} more" if more > 0 else "") + lines.append(f"{severity.value}: {text} ({len(tasks)} tasks: {shown}; agent-env task lint lists each)") + return lines + + +def unpinned_summary(findings: Iterable[tuple[str, Finding]]) -> str | None: + """``findings``' unpinned references as one line, by how many tasks name how many of each kind: an eval of + thousands of tasks naming the latest of their agents and artifacts, as the hub's do, would otherwise say one line + for each, since each names its own step. None when there are none.""" + tasks, kinds = set(), collections.Counter() + for task, finding in findings: + if finding.rule == UNPINNED: + tasks.add(task) + kinds[finding.about] += 1 + if not tasks: + return None + total = sum(kinds.values()) + counts = ", ".join(f"{kind} {n:,}" for kind, n in kinds.most_common()) + return (f"warning: {len(tasks):,} task{'s' * (len(tasks) != 1)} name{'s' * (len(tasks) == 1)} {total:,} store " + f"entit{'ies' if total != 1 else 'y'} with no version, so each run takes the latest ({counts}); " + "agent-env task lint --id TASK lists a task's") + + +def refuses(finding: Finding, *, skip_lint: bool = False) -> bool: + """Whether ``finding`` refuses a run. ``skip_lint`` lifts lint's own rules, not a step's ``config_problems``, which + refused a run before lint did and fail it when it runs.""" + return finding.severity is Severity.ERROR and not (skip_lint and finding.rule != "step-config") + + +def errors(findings: Iterable[Finding]) -> list[Finding]: + return [finding for finding in findings if finding.severity is Severity.ERROR] + + +def is_verifier(step: TaskStep) -> bool: + """A step that grades: one on the VerifierTaskStep base, or one that still writes its own legacy entry, a grader (as + agent-env's validators are) or a step with a ``verifier_id`` (as agentenvhub-sdk's are).""" + return (isinstance(step, VerifierTaskStep) or step.step_role is StepRole.GRADER + or bool(getattr(step, "verifier_id", None))) + + +def _own(step: TaskStep) -> bool: + """One of agent-env's own steps, whose behaviour lint knows.""" + return type(step).__module__.partition(".")[0] == "agent_env" + + +def _seen_into(step: TaskStep) -> bool: + """A step whose verification writes lint knows: one on the base writes under its ``verifier_id``, and another of + agent-env's own writes none unless it's a grader, whose keys are its own. An outside step off the base may write + any key, as agentenvhub-sdk's ``run_lighthouse_audit`` writes under its ``verification_id``.""" + return isinstance(step, VerifierTaskStep) or (_own(step) and step.step_role is not StepRole.GRADER) + + +class _Reads(dict): + """A step dict that records each key its type's ``from_dict`` asks for, read or absent; spreading or iterating it + counts as reading every key.""" + + def __init__(self, data: Mapping[str, Any]): + super().__init__(data) + self.asked: set[str] = set() + + def __getitem__(self, key): + self.asked.add(key) + return super().__getitem__(key) + + def get(self, key, default=None): + self.asked.add(key) + return super().get(key, default) + + def __contains__(self, key): + self.asked.add(key) + return super().__contains__(key) + + def pop(self, key, *default): + self.asked.add(key) + return super().pop(key, *default) + + def setdefault(self, key, default=None): + self.asked.add(key) + return super().setdefault(key, default) + + def __iter__(self): + self.asked.update(super().keys()) + return super().__iter__() + + def keys(self): + self.asked.update(super().keys()) + return super().keys() + + def items(self): + self.asked.update(super().keys()) + return super().items() + + def values(self): + self.asked.update(super().keys()) + return super().values() + + +def unknown_fields(step_dicts: Sequence[Mapping[str, Any]]) -> Iterator[Finding]: + """Each key of a step that its type never asks for when it reads the step, with the nearest field it asks for.""" + registry = get_task_step_registry() + for step in step_dicts: + cls = registry.get(step.get("type")) + if cls is None: + continue + tracked = _Reads(copy.deepcopy(dict(step))) + try: + attach_base_fields(cls.from_dict(tracked), tracked) + except Exception: + continue # a step that can't be read is reported where the task is built + for key in sorted(set(step) - tracked.asked - {"type"}): + near = difflib.get_close_matches(key, sorted(tracked.asked - set(step)), n=1) + hint = f" (did you mean {near[0]!r}?)" if near else "" + yield Finding(Severity.ERROR, "unknown-field", + f"{cls.type} doesn't read {key!r}{hint}, so it would be ignored", step.get("id")) + + +def _step_configs(steps: Sequence[TaskStep]) -> Iterator[Finding]: + for step in steps: + for problem in step.config_problems(): + yield Finding(Severity.ERROR, "step-config", problem, step.id) + + +def _step_warnings(steps: Sequence[TaskStep]) -> Iterator[Finding]: + for step in steps: + for doubt in step.config_warnings(): + yield Finding(Severity.WARNING, "step-config", doubt, step.id) + + +def _headlines(task: Task) -> Iterator[Finding]: + by_id = {step.id: step for step in task.steps} + for headline in task.headline_verifiers or (): + step = by_id.get(headline) + if step is None: + continue # Task refuses it when it's built + if isinstance(step, VerifierTaskStep): + continue + if _own(step) and not is_verifier(step): + yield Finding(Severity.ERROR, "headline-not-a-verifier", + f"headline_verifiers names {headline!r}, a {step.type} step, which grades nothing") + else: + yield Finding(Severity.WARNING, "headline-without-grade", + f"headline_verifiers names {headline!r}, a {step.type} step that writes no named grade, " + "so a run's result will read it as ungraded") + + +def _aggregate_sources(steps: Sequence[TaskStep]) -> Iterator[Finding]: + """An aggregate source that no verifier in the task writes, which fails the aggregate when it grades: an error when + lint sees every key the task's steps write, else a warning naming the steps it can't see into.""" + written, unseen = set(), [step.id for step in steps if not _seen_into(step)] + for step in steps: + if isinstance(getattr(step, "verifier_id", None), str): + written.add(step.verifier_id) + if (isinstance(step, RubricsVerifierTaskStep) + and step.output_format is JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES): + written.add(trajectory_mistakes_key(step.verifier_id)) + for step in steps: + if isinstance(step, AggregateVerifiersTaskStep): + missing = [source for source in step.verifier_ids if source not in written] + if missing and not unseen: + yield Finding(Severity.ERROR, "aggregate-source", + f"verifier_ids names {missing!r}, which no verifier in this task writes (it has " + f"{sorted(written)})", step.id) + elif missing: + yield Finding(Severity.WARNING, "aggregate-source", + f"verifier_ids names {missing!r}, which no verifier in this task names as its " + f"verifier_id; unless {unseen!r} write it, under keys lint can't see, the aggregate fails " + "when it grades", step.id) + + +def _unpinned(steps: Sequence[TaskStep]) -> Iterator[Finding]: + """A store entity a step reads with no version, so each run takes whatever is latest. A bundle's own entities and + installed bundles' are local ids, which their bundles write and pin.""" + for step in steps: + for site in ref_sites(type(step).entity_refs or (), step.to_dict()): + if (site.ref.role is RefRole.INPUT and site.version_key is not None and site.version is None + and isinstance(site.value, str) and site.value and not is_local_id(site.value)): + yield Finding(Severity.WARNING, UNPINNED, + f"{site.path} names {site.ref.kind.value} {site.value!r} with no version, so each run " + f"takes its latest; set {site.version_key} to pin it", step.id, site.ref.kind.value) + + +def _coverage(task: Task) -> Iterator[Finding]: + verifiers = [step for step in task.steps if is_verifier(step)] + if not verifiers and all(_own(step) for step in task.steps): + yield Finding(Severity.WARNING, "no-verifier", "no step grades this task, so a run has no score") + elif len(verifiers) > 1 and not task.headline_verifiers: + yield Finding(Severity.WARNING, "no-headlines", _no_headlines(verifiers)) + + +def _no_headlines(verifiers: list[TaskStep]) -> str: + """Several verifiers and no headline: every grade is top-line. Name the aggregate that covers the others, if one + does.""" + ids = {step.verifier_id for step in verifiers if isinstance(getattr(step, "verifier_id", None), str)} + covering = [step.id for step in verifiers if isinstance(step, AggregateVerifiersTaskStep) + and ids - {step.verifier_id} <= set(step.verifier_ids)] + suggestion = (f"; {covering[0]!r} aggregates the others, so name it in headline_verifiers" if covering + else "; name the one that is its reward in headline_verifiers") + return (f"{len(verifiers)} verifiers and no headline_verifiers, so every grade is top-line and anything that needs " + f"one number must name one{suggestion}") diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 67445e1e8..5bb8abdfb 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -8,7 +8,7 @@ from collections.abc import Callable from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Any +from typing import Any, Optional, TypedDict from agentenv_protocol import client as protocol_v1 @@ -235,6 +235,15 @@ def sandbox_overrides(sandbox: str) -> dict[str, str]: ) +class TurnTrajectory(TypedDict): + """What a turn's trajectory reads left, beside its native copy's url in the per-turn list: the native copy's + format, as the agent's answer named it (None from an agent that names none), and its atif/1 copy, None when the + agent offers none.""" + + native_format: Optional[str] + atif: Optional[dict[str, Any]] + + @dataclass class PromptResponse: prompt_id: str @@ -259,6 +268,8 @@ class PromptResponse: target_agent_per_turn_trajectory_object_urls: list[str | None] | None = None # The step's usage, one entry per role that ran turns in it (``task_step.usage``); None before it was recorded. usage: list[dict] | None = None + # Each turn's ``TurnTrajectory``, beside ``target_agent_per_turn_trajectory_object_urls``. + target_agent_per_turn_trajectories: list[TurnTrajectory] | None = None def to_dict(self) -> dict[str, Any]: """The stored form: every field, plus the legacy key of each renamed one.""" @@ -294,6 +305,7 @@ def from_dict(cls, data: dict) -> PromptResponse: step_id=data.get("step_id"), structured_output=data.get("structured_output"), usage=data.get("usage"), + target_agent_per_turn_trajectories=data.get("target_agent_per_turn_trajectories"), ) diff --git a/src/agent_env/task_step/snapshot_utils/live_trajectory.py b/src/agent_env/task_step/snapshot_utils/live_trajectory.py index f14ccb723..33176777e 100644 --- a/src/agent_env/task_step/snapshot_utils/live_trajectory.py +++ b/src/agent_env/task_step/snapshot_utils/live_trajectory.py @@ -89,9 +89,11 @@ def __init__( poll_interval_seconds: float, previous_turn_id: str | None, headers: Mapping[str, str], + format: str | None = None, ) -> None: self._endpoint = endpoint self._headers = headers + self._format = format self._store = store self._prefix_key = prefix_key self._turn_id = turn_id @@ -115,7 +117,7 @@ def follow(self, task_id: str) -> None: asyncio.create_task( follow_trajectory( self._endpoint, task_id, sink, poll_interval_seconds=self._poll_interval_seconds, stop=self.stop, - headers=self._headers, + headers=self._headers, format=self._format, ) ) ) @@ -135,6 +137,7 @@ async def following_live_trajectory( poll_interval_seconds: float, headers: Mapping[str, str], previous_turn_id: str | None = None, + format: str | None = None, ) -> AsyncIterator[LiveTurn]: """Yield the turn whose task is followed once its message is sent; then ``previous_turn_id``'s ``next.json`` names it. @@ -144,7 +147,7 @@ async def following_live_trajectory( end the follower did not read is marked ended in the state the body reported, else ``canceled`` for a cancelled step or ``failed``, so readers do not wait on it. A follower's own failure is logged and never fails the turn.""" - turn = LiveTurn(endpoint, store, prefix_key, turn_id, poll_interval_seconds, previous_turn_id, headers) + turn = LiveTurn(endpoint, store, prefix_key, turn_id, poll_interval_seconds, previous_turn_id, headers, format) stopped_as = TrajectoryState.FAILED try: @@ -184,9 +187,10 @@ class LiveTurns: ``next.json``, and ``end`` marks the last turn sent as the step's last.""" def __init__(self, endpoint: str, store: ObjectStore, prefix_key: str, *, poll_interval_seconds: float, - headers: Mapping[str, str]) -> None: + headers: Mapping[str, str], format: str | None = None) -> None: self._endpoint = endpoint self._headers = headers + self._format = format self._store = store self._prefix_key = prefix_key self._poll_interval_seconds = poll_interval_seconds @@ -203,6 +207,7 @@ async def following(self, turn_id: str) -> AsyncIterator[LiveTurn]: poll_interval_seconds=self._poll_interval_seconds, headers=self._headers, previous_turn_id=self._last_sent, + format=self._format, ) as turn: try: yield turn diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index 1540d5ee4..141b77221 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -158,6 +158,11 @@ def config_problems(self) -> list[str]: ``preflight`` is, and also for a step whose preflight a dry run skips.""" return [] + def config_warnings(self) -> list[str]: + """Doubts about this step's own fields that don't stop it running, found without reading a store; empty when + fine. Task lint reports them as warnings, as it reports ``config_problems`` as errors.""" + return [] + def task_problems(self, steps: list["TaskStep"]) -> list[str]: """Problems this step has with the task's other steps (``steps``: all of them, this one included, in order), found without reading a store; empty when fine. A task with any is refused when it loads.""" diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py index 899d05fd9..2098450d1 100644 --- a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -66,7 +66,7 @@ TURN_GRACE_SECONDS, AlwaysOnTaskStep, ) -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, fetch_native_trajectory, is_prompt_step logger = logging.getLogger(__name__) @@ -268,11 +268,11 @@ async def _trajectory(self, agent: DeployedAgent, session: _Session, task_id: st if not ext or not session.trajectory_prefix: return None try: - # prompt_agent's fetch uses nothing of its step, so the run's own code stores it. - return await asyncio.wait_for(PromptAgentTaskStep._fetch_trajectory( - None, agent.a2a_url or agent.api_url, ext, task_id, session.trajectory_prefix, message_id, + stored = await asyncio.wait_for(fetch_native_trajectory( + agent.a2a_url or agent.api_url, ext, task_id, session.trajectory_prefix, message_id, card=card, sandbox_type=agent.sandbox_type, headers=agent.control_headers), max(0.0, min(TRAJECTORY_FETCH_SECONDS, deadline - time.monotonic()))) + return stored.url except Exception as e: logger.warning(f"user sim trajectory fetch failed (continuing): {e}") return None diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index f32c60b46..84f75763b 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -118,7 +118,11 @@ from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, user_sim_model from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import ( + PromptAgentTaskStep, + fetch_native_trajectory, + prescribe_formats, +) from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc from agent_env.task_step.task_steps.agent.user_sim_common import attached, file_name, for_agent, recorded_opening @@ -597,6 +601,7 @@ async def _configure_agents(self) -> None: if negotiated: await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields, headers=self.agent.control_headers) + await prescribe_formats(ctx, step.id, self.agent, self.target_url) solver_model = negotiated.fields.get("model") if negotiated else None ctx.record_model(ModelRole.SOLVER.record_key, step.id, solver_model) self.usage.start(ModelRole.SOLVER, solver_model, self.agent) @@ -1287,10 +1292,10 @@ def record() -> None: traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if traj_ext: try: - traj = await asyncio.wait_for( - step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, - card=card, sandbox_type=self.agent.sandbox_type, - headers=self.agent.control_headers), TRAJECTORY_FETCH_SECONDS) + traj = (await asyncio.wait_for( + fetch_native_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, + card=card, sandbox_type=self.agent.sandbox_type, + headers=self.agent.control_headers), TRAJECTORY_FETCH_SECONDS)).url except Exception as e: logger.warning(f"wake {self.wakes}: trajectory fetch failed (continuing): {e}") self.per_turn_trajectory_urls.append(traj) @@ -1368,10 +1373,10 @@ def record() -> None: traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if traj_ext: try: - traj = await asyncio.wait_for( - step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, - card=card, sandbox_type=self.agent.sandbox_type, - headers=self.agent.control_headers), TRAJECTORY_FETCH_SECONDS) + traj = (await asyncio.wait_for( + fetch_native_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, + card=card, sandbox_type=self.agent.sandbox_type, + headers=self.agent.control_headers), TRAJECTORY_FETCH_SECONDS)).url except Exception as e: logger.warning(f"nudge after wake {self.wakes}: trajectory fetch failed (continuing): {e}") self.per_turn_trajectory_urls.append(traj) @@ -1584,12 +1589,12 @@ async def _ask_user_sim(self, text: str, deadline: float) -> dict: traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if traj_ext: try: - traj = await asyncio.wait_for( - self.step._fetch_trajectory(self.user_url, traj_ext, user_task_id, - self.step._trajectory_prefix() + "user_sim/", msg_id, - card=card, sandbox_type=self.user_agent.sandbox_type, - headers=self.user_agent.control_headers), - max(0.0, min(TRAJECTORY_FETCH_SECONDS, deadline - time.monotonic()))) + traj = (await asyncio.wait_for( + fetch_native_trajectory(self.user_url, traj_ext, user_task_id, + self.step._trajectory_prefix() + "user_sim/", msg_id, + card=card, sandbox_type=self.user_agent.sandbox_type, + headers=self.user_agent.control_headers), + max(0.0, min(TRAJECTORY_FETCH_SECONDS, deadline - time.monotonic())))).url except Exception as e: logger.warning(f"user sim trajectory fetch failed (continuing): {e}") return {"text": raw, "structured": structured if isinstance(structured, dict) else None, diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 24cf395fd..b2de5b964 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -11,7 +11,7 @@ from contextlib import nullcontext from datetime import datetime, timezone from collections.abc import Mapping -from typing import Any, ClassVar, Optional +from typing import Any, ClassVar, NamedTuple, Optional import httpx from a2a.types import TaskState from agentenv_protocol.a2a_agent import card_request_accepts @@ -19,6 +19,7 @@ from agent_env.a2a_agent import A2AAgent, conversation_store from agent_env.a2a_agent import protocol from agent_env.a2a_agent.object_transfer import ( + TrajectoryFormats, TrajectoryUpload, fetch_trajectory, send_and_wait, @@ -36,7 +37,13 @@ ) from agent_env.store import DuplicateKeyError, get_config from agent_env.store.object_store import ObjectStore -from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.context import ( + DeployedAgent, + PromptResponse, + StepContextError, + TaskStepContext, + TurnTrajectory, +) from agent_env.task_step.usage import StepUsage from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope @@ -560,6 +567,7 @@ async def _converse( if negotiated: await protocol.post_agent_config(target_url + negotiated.endpoint, negotiated.fields, headers=agent.control_headers) + await prescribe_formats(context, self.id, agent, target_url) # A model the agent's card doesn't accept is never sent, and the agent runs its own. solver_model = negotiated.fields.get("model") if negotiated else None context.record_model(ModelRole.SOLVER.record_key, self.id, solver_model) @@ -633,11 +641,14 @@ async def _converse( conversation_tool_call_count: Optional[int] = None final_task_id = "" per_turn_trajectory_urls: list[str | None] = [] + per_turn_trajectories: list[TurnTrajectory] = [] source_agent_per_turn_prompt_parts: list[list[dict] | None] = [] traj_ext_cached = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + formats = TrajectoryFormats.of(traj_ext_cached) live = self._live_trajectory_target(context, card, target_url, trajectory_output_prefix) live_turns = ( - LiveTurns(*live, poll_interval_seconds=self.poll_interval_seconds, headers=agent.control_headers) + LiveTurns(*live, poll_interval_seconds=self.poll_interval_seconds, headers=agent.control_headers, + format=formats.native) if live is not None else None ) final_state: str = TaskState.completed.value @@ -710,16 +721,17 @@ def record_turn() -> None: ) if traj_ext_cached: - turn_traj_uri: str | None = None + stored = StoredTrajectory() try: - turn_traj_uri = await self._fetch_trajectory( + stored = await fetch_native_trajectory( target_url, traj_ext_cached, sent_task_id, trajectory_output_prefix, target_a2a_task_id, card=card, sandbox_type=agent.sandbox_type, headers=agent.control_headers, ) except Exception as e: logger.warning(f"Turn {turn+1} trajectory fetch failed (continuing): {e}") - per_turn_trajectory_urls.append(turn_traj_uri) + per_turn_trajectory_urls.append(stored.url) + per_turn_trajectories.append(TurnTrajectory(native_format=stored.format, atif=None)) final_task_id = sent_task_id final_state = target_state @@ -841,6 +853,7 @@ def record_turn() -> None: agent_trajectory_object_url=trajectory_url, agent_trajectory_object_prefix=trajectory_output_prefix, target_agent_per_turn_trajectory_object_urls=per_turn_trajectory_urls or None, + target_agent_per_turn_trajectories=per_turn_trajectories or None, source_agent_per_turn_prompt_parts=source_agent_per_turn_prompt_parts or None, tool_call_count=conversation_tool_call_count, model=model, @@ -901,30 +914,6 @@ def _live_trajectory_target( return None return endpoint, store, prefix_key - async def _fetch_trajectory( - self, a2a_url: str, traj_ext: dict, a2a_server_task_id: str, trajectory_output_prefix: str, - target_a2a_task_id: str, *, sandbox_type: str | None, headers: Mapping[str, str], card: dict | None = None, - ) -> str | None: - get_method, get_path = A2AAgent.operation(traj_ext, "get") - store = transfer_store(get_config().get_object_store(), a2a_url, card, sandbox_type=sandbox_type) - mode = trajectory_mode(get_method, store, by="task_id", sandbox_type=sandbox_type) - if mode is None: - raise RuntimeError( - "Agent advertises no trajectory get form this object store can serve" - ) - # Fetched by a2a_server_task_id, the id the A2A server produced; stored under - # target_a2a_task_id, the client message id persisted on the conversation as - # a2a_task_id, so the FE can resolve it. - upload = None - if mode == "objects": - upload = await asyncio.to_thread( - TrajectoryUpload.to, - store, trajectory_object_url(trajectory_output_prefix, name=target_a2a_task_id, store=store), - ) - fetched = await fetch_trajectory(a2a_url + get_path, {"task_id": a2a_server_task_id}, upload=upload, store=store, - headers=headers) - return await asyncio.to_thread(store_trajectory, fetched, trajectory_output_prefix, name=target_a2a_task_id) - async def _env_clock(self, envs: list) -> dict: """``{"env_clock": [...]}``, each armed env clock for a conversation message, or {} when no clock is armed.""" if not envs: @@ -1190,13 +1179,13 @@ async def _user_sim_trajectory(self, task_id: str, message_id: str) -> Optional[ ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) if not ext: return None - step = self._step try: - return await asyncio.wait_for( - step._fetch_trajectory(self._url, ext, task_id, step._trajectory_prefix() + "user_sim/", message_id, - card=card, sandbox_type=self._user_sim.sandbox_type, - headers=self._user_sim.control_headers), + stored = await asyncio.wait_for( + fetch_native_trajectory(self._url, ext, task_id, self._step._trajectory_prefix() + "user_sim/", + message_id, card=card, sandbox_type=self._user_sim.sandbox_type, + headers=self._user_sim.control_headers), _USER_SIM_TRAJECTORY_FETCH_SECONDS) + return stored.url except Exception as e: logger.warning(f"user sim trajectory fetch failed (continuing): {e}") return None @@ -1370,6 +1359,103 @@ async def _record(self, number: int, read: dict, ans: dict, calls: list[dict], e # How long one user sim trajectory fetch may take, as in always_on. _USER_SIM_TRAJECTORY_FETCH_SECONDS = 120.0 +#: How long an agent may take to answer a trajectory prescription, and how often one that can't be reached is tried, +#: the wait growing by this many seconds each time. +_PRESCRIBE_TIMEOUT_SECONDS = 30.0 +_PRESCRIBE_ATTEMPTS = 3 +_PRESCRIBE_RETRY_SECONDS = 1.0 +#: Where each step records the trajectory formats it prescribed and what its agent said it builds. +TRAJECTORY_FORMATS_KEY = "trajectory_formats" + + +def prescribed_formats(context: TaskStepContext) -> tuple[str, ...]: + """The trajectory formats beside native the run prescribes to each agent whose trajectory it reads: its + ``user_overrides.trajectory_formats``, else ``[trajectories] formats``; none means native only.""" + formats = context.metadata.get("user_overrides", {}).get("trajectory_formats") + if formats is None: + return get_config().get_trajectory_formats() + if not isinstance(formats, list) or not all(isinstance(format, str) for format in formats): + raise StepContextError(f"user_overrides.trajectory_formats must be a list of format names, got {formats!r}") + return tuple(formats) + + +async def prescribe_formats(context: TaskStepContext, step_id: str, agent: DeployedAgent, a2a_url: str) -> None: + """Have the agent build the run's trajectory formats (``prescribed_formats``) beside native for the conversations it + starts from now on, when its card offers the trajectory extension's ``prescribe``; an agent whose SDK predates it + builds what it always has. What was prescribed and what the agent builds go into + ``metadata["trajectory_formats"][step_id]``, and a prescribed format the agent can't build is warned of. + + A connection error is retried; after a failure the agent builds what it was last prescribed, native only unless an + earlier step prescribed more, and the run goes on: its judges read native either way.""" + formats = prescribed_formats(context) + extension = A2AAgent.find_extension(agent.a2a_card or {}, A2AAgent.EXT_TRAJECTORY) + method, path = A2AAgent.operation(extension, "prescribe") + offered = TrajectoryFormats.of(extension).formats + record: dict[str, Any] = {"agent": agent.agent_name, "prescribed": list(formats), "built": None} + context.metadata.setdefault(TRAJECTORY_FORMATS_KEY, {})[step_id] = record + unbuildable = [format for format in formats if format not in offered] + if unbuildable: + logger.warning(f"agent '{agent.agent_name}' can't build {unbuildable}, which the run prescribes: it builds " + f"{list(offered) if offered else 'only its native trajectory'}") + if method is None or path is None: + return + wanted = [format for format in formats if format in offered] + for attempt in range(1, _PRESCRIBE_ATTEMPTS + 1): + try: + async with httpx.AsyncClient() as client: + response = await client.post(a2a_url + path, json={"formats": wanted}, headers=agent.control_headers, + timeout=_PRESCRIBE_TIMEOUT_SECONDS) + protocol.raise_for_extension_status(response, operation="trajectory prescribe") + record["built"] = response.json()["formats"] + return + except httpx.TransportError as e: + if attempt < _PRESCRIBE_ATTEMPTS: + await asyncio.sleep(_PRESCRIBE_RETRY_SECONDS * attempt) + continue + failure: Exception = e + except (httpx.HTTPError, ValueError, KeyError) as e: + failure = e + record["error"] = f"{type(failure).__name__}: {failure}" + logger.warning(f"agent '{agent.agent_name}': prescribing trajectory formats {wanted} failed, so it builds what " + f"it was last prescribed (continuing): {failure}") + return + + +class StoredTrajectory(NamedTuple): + """Where a turn's native trajectory was stored, None when the agent returned none, and the format its answer + named, None when it named none.""" + + url: str | None = None + format: str | None = None + + +async def fetch_native_trajectory( + a2a_url: str, traj_ext: dict, a2a_server_task_id: str, trajectory_output_prefix: str, target_a2a_task_id: str, + *, sandbox_type: str | None, headers: Mapping[str, str], card: dict | None = None, +) -> StoredTrajectory: + """A turn's native trajectory, read by ``a2a_server_task_id`` and stored under ``target_a2a_task_id``. A card that + offers formats is asked for its native one by name, so the copy judges read stays native whatever the card's + default; one that offers none gets the read it always has.""" + get_method, get_path = A2AAgent.operation(traj_ext, "get") + store = transfer_store(get_config().get_object_store(), a2a_url, card, sandbox_type=sandbox_type) + mode = trajectory_mode(get_method, store, by="task_id", sandbox_type=sandbox_type) + if mode is None: + raise RuntimeError( + "Agent advertises no trajectory get form this object store can serve" + ) + # Fetched by a2a_server_task_id, the id the A2A server produced; stored under + # target_a2a_task_id, the client message id persisted on the conversation as + # a2a_task_id, so the FE can resolve it. + upload = None + if mode == "objects": + upload = await asyncio.to_thread( + TrajectoryUpload.to, + store, trajectory_object_url(trajectory_output_prefix, name=target_a2a_task_id, store=store), + ) + fetched = await fetch_trajectory(a2a_url + get_path, {"task_id": a2a_server_task_id}, upload=upload, store=store, + headers=headers, format=TrajectoryFormats.of(traj_ext).native) + url = await asyncio.to_thread(store_trajectory, fetched, trajectory_output_prefix, name=target_a2a_task_id) + return StoredTrajectory(url, fetched.format) def _agent_text(solver_message: str, agent_parts: Optional[list[dict]]) -> str: diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index 64284ea91..44ebce964 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -536,7 +536,7 @@ async def _export_environment_to_file( if base_url is None and not mcp: return None if base_url is not None: - headers = deployed.control_headers_for(base_url) if deployed is not None else None + headers = (deployed.control_headers_for(base_url) or None) if deployed is not None else None if returns_file_tree(card or {}): upload = await asyncio.to_thread(_snapshot_upload, card or {}, timeout_seconds, getattr(deployed, "sandbox_type", None), FILE_TREE_LIMITS) @@ -582,6 +582,8 @@ async def _export_environment_to_file( raise ValueError( f"FilePart for '{environment_name}' names {uri!r}, which is not on its service's address {base_url}" ) + # The service names this url, so it gets the token only where it resolves under the service. + uri_headers = protocol_v1.granted_headers(uri, {base_url: protocol_v1.request_headers(base_url, headers)}) async with httpx.AsyncClient() as client: async with client.stream( "GET", @@ -589,6 +591,7 @@ async def _export_environment_to_file( timeout=httpx.Timeout( timeout_seconds, connect=_CONNECT_TIMEOUT_SECONDS ), + headers=uri_headers, ) as resp_file: resp_file.raise_for_status() with open(tmp_path, "wb") as f: @@ -601,11 +604,13 @@ async def _export_environment_to_file( return ".json" legacy_url = f"{legacy_protocol.environment_base_url(gateway_url, environment_name, mcp=True)}/export-state" + legacy_headers = (deployed.control_headers_for(legacy_url) or None) if deployed is not None else None async with httpx.AsyncClient() as client: async with client.stream( "GET", legacy_url, timeout=httpx.Timeout(timeout_seconds, connect=_CONNECT_TIMEOUT_SECONDS), + headers=protocol_v1.request_headers(legacy_url, legacy_headers), ) as resp: resp.raise_for_status() seen_first_bytes = False diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index d27084736..6ea92b35e 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -20,7 +20,7 @@ import logging import re import uuid -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.task_step.task_step import TaskStepDependency @@ -65,6 +65,14 @@ def to_dict(self) -> dict: base["verifier_id"] = self.verifier_id return base + def config_problems(self) -> list[str]: + """The criteria a run would refuse when it grades: an unknown type, an empty ``needles`` or ``pattern``, a + pattern that doesn't compile, or a flag ``re`` doesn't have.""" + problems = super().config_problems() + for idx, criterion in enumerate(self.criteria): + problems.extend(f"criterion #{idx}: {problem}" for problem in _criterion_problems(criterion)) + return problems + @classmethod def from_dict(cls, data: dict) -> "AgentPromptResponseVerifierTaskStep": return cls(**cls._base_from_dict(data), prompt_id=data["prompt_id"], criteria=data.get("criteria")) @@ -143,3 +151,32 @@ def _eval_criterion(self, idx: int, response: str, criterion: dict) -> dict: "justification": "pattern matched" if passed else f"no match for pattern {pattern!r}", } raise RuntimeError(f"unhandled criterion type {rtype}") + + +def _criterion_problems(criterion: Any) -> list[str]: + if not isinstance(criterion, dict): + return [f"must be an object, not {type(criterion).__name__}"] + rtype = criterion.get("type") + if not isinstance(rtype, str) or rtype not in _HANDLED_TYPES: + return [f"unknown type {rtype!r}; handled types: {sorted(_HANDLED_TYPES)}"] + if rtype == "response_contains": + needles = criterion.get("needles") + if isinstance(needles, list) and needles and all(isinstance(needle, str) for needle in needles): + return [] + return [f"response_contains needs non-empty needles, a list of strings, not {needles!r}"] + pattern = criterion.get("pattern") + if not (isinstance(pattern, str) and pattern): + return [f"response_regex_present needs a non-empty pattern, a string, not {pattern!r}"] + names = criterion.get("flags") + if names is not None and not (isinstance(names, list) and all(isinstance(name, str) for name in names)): + return [f"flags must be a list of re flag names, not {names!r}"] + if unknown := [name for name in names or () if not isinstance(getattr(re, name, None), re.RegexFlag)]: + return [f"flags {unknown!r} aren't re flags"] + flags = re.DOTALL if names is None else 0 + for name in names or (): + flags |= getattr(re, name) + try: + re.compile(pattern, flags) + except re.error as e: + return [f"pattern {pattern!r} doesn't compile: {e}"] + return [] diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 9efd02b3c..8743700cf 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -3,13 +3,11 @@ from __future__ import annotations import asyncio -import contextlib -import io import json import logging import re import uuid -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from typing import Any, ClassVar, Optional @@ -22,7 +20,8 @@ from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config from agent_env.config.errors import ConfigError -from agent_env.config.model import ModelParam, ModelRole, api_root, no_key_for +from agent_env.config import model_call +from agent_env.config.model import ModelRole, no_key_for from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import ( PromptResponse, @@ -283,39 +282,6 @@ def _caches_prompts(model: str) -> bool: return "claude" in model.lower() -def _judge_route(model: str, api_base: Optional[str], provider_of: Callable[..., Any], - custom_provider: Optional[str] = None) -> tuple[str, Optional[str]]: - """The model name and base a direct judge call hands litellm; what it records stays ``model``. Through an - endpoint, a bare name goes by what litellm makes of it: an Anthropic one keeps that route, with the base's - trailing ``/v1`` dropped since its client appends ``/v1/messages``; an OpenAI one goes as configured; any other, - which litellm would send to that provider's own API (``gemini-*`` to Vertex), or one it can't place, goes - through the endpoint as ``openai/``. A provider prefix in the name, or a ``custom_llm_provider`` among the - call's params, keeps the route it names. ``provider_of`` is litellm's ``get_llm_provider``, which prints a banner - before refusing a name it can't place.""" - if api_base is None: - return model, None - try: - with contextlib.redirect_stdout(io.StringIO()): - provider = provider_of(model, custom_llm_provider=custom_provider)[1] - except Exception: - return f"openai/{model}", api_base - if provider == "anthropic": - return model, api_root(api_base) - if provider == "openai" or custom_provider or "/" in model: - return model, api_base - return f"openai/{model}", api_base - - -def _usage_and_cost(response: Any) -> dict: - """A LiteLLM response's token counts and LiteLLM's price of it in USD, each None when it has none.""" - usage = getattr(response, "usage", None) - usage = usage.model_dump() if hasattr(usage, "model_dump") else usage - hidden = getattr(response, "_hidden_params", None) - cost = hidden.get("response_cost") if isinstance(hidden, dict) else None - return {"usage": usage if isinstance(usage, dict) else None, - "cost": float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None} - - def _image_url_block(frame: ImageFrame) -> dict: """Adapt a neutral trajectory image to LiteLLM's image block shape.""" return { @@ -1393,9 +1359,10 @@ async def _prompt_llm_judge( system: Optional[str] = None, cached_prefix: Optional[str] = None, ) -> dict: - """Direct LiteLLM judge — no agent boot, no tools. Returns {response, trajectory_object_url, usage, cost}: - ``usage`` the call's token counts and ``cost`` LiteLLM's price of it in USD, each None when the response - has none, for a caller that records them (this step records neither). + """Direct judge — no agent boot, no tools — through core's one model call (``config.model_call``). Returns + ``response`` and ``trajectory_object_url`` (None), and the call's record for a caller that keeps it (this step + doesn't): its ``usage`` as the response carried it, the ``model`` called, the ``endpoint_host`` it reached, and + its cost's keys, ``cost_usd`` what the endpoint billed or else LiteLLM's price map's figure. When ``image_blocks`` are provided the user message becomes multimodal (text + images); otherwise it stays a plain string. ``system`` is sent as a system message first. ``cached_prefix`` is the start of @@ -1403,20 +1370,11 @@ async def _prompt_llm_judge( marked for Anthropic's prompt cache, so the calls after the first read it from the cache; a prompt that doesn't start with it is sent whole. The step's own calls pass neither, so their request is unchanged; a subclass opts in by overriding this and calling it with them.""" - import litellm - - from agent_env.config import get_config - overrides = context.metadata.get("user_overrides", {}) - config = get_config() - api_key = config.resolve_reference(overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key")) - base = overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url") or self.default_model_api_base - if base and not api_key and not config.is_model_endpoint(base): - raise ConfigError(f"{self.type} '{self.id}': {no_key_for(base, _JUDGE_KEY_HOW)}") - call_config = config.resolve_model_call(model, default_api_key=api_key, base_override=base) - route_model, route_base = _judge_route(model, call_config.api_base, litellm.get_llm_provider, - call_config.params.get("custom_llm_provider")) - call_config = replace(call_config, api_base=route_base) + route = model_call.resolve_route( + model, api_key=overrides.get("judge_litellm_api_key") or overrides.get("litellm_api_key"), + api_base=overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url") + or self.default_model_api_base, who=f"{self.type} '{self.id}'", key_how=_JUDGE_KEY_HOW) text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}] cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model) if cached: @@ -1441,17 +1399,9 @@ async def _prompt_llm_judge( last_exc: Exception | None = None for attempt in range(1, self.DEFAULT_MAX_RETRIES + 1): try: - request_kwargs: dict = dict( - model=route_model, - messages=messages, - timeout=self.judge_timeout_seconds, - **call_config.client_kwargs(), - ) - if use_response_format: - request_kwargs[ModelParam.RESPONSE_FORMAT] = response_format - response = await litellm.acompletion(**request_kwargs) - content = response.choices[0].message.content or "" - return {"response": content, "trajectory_object_url": None, **_usage_and_cost(response)} + reply = await model_call.complete(route, messages, timeout=self.judge_timeout_seconds, + response_format=response_format if use_response_format else None) + return {"response": reply.content, "trajectory_object_url": None, **reply.fields()} except Exception as exc: last_exc = exc if image_blocks and use_response_format and _looks_like_response_format_error(exc): diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 4f0cdd36e..897ebf1aa 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -145,6 +145,28 @@ def __init__( raise ValueError(f"run_container_unit_tests_verifier '{id}': result paths in env '{env_id}' are relative " f"to its session user's home, not {absolute}") + def config_problems(self) -> list[str]: + """A ``command`` or ``timeout_sec`` a run can't use, and a negative ``timeout_sec``, which ``timeout`` + refuses, so the tests never run.""" + problems = super().config_problems() + if not isinstance(self.command, str): + problems.append(f"command must be a string, not {type(self.command).__name__}") + if not isinstance(self.timeout_sec, (int, float)) or isinstance(self.timeout_sec, bool): + problems.append(f"timeout_sec must be a number of seconds, not {self.timeout_sec!r}") + elif self.timeout_sec < 0: + problems.append(f"timeout_sec {self.timeout_sec} is negative; give the tests at least 1 s") + return problems + + def config_warnings(self) -> list[str]: + """What runs without testing anything: an empty ``command``, which exits 0 and so passes without a + ``reward_path``, and a ``timeout_sec`` of 0, which ``timeout`` reads as no limit.""" + warnings = super().config_warnings() + if isinstance(self.command, str) and not self.command.strip() and self.reward_path is None: + warnings.append("command is empty, so it exits 0 and the tests pass without running anything") + if self.timeout_sec == 0: + warnings.append("timeout_sec 0 means no time limit, so a hung test runs until the sandbox's own TTL") + return warnings + def to_dict(self) -> dict: base = super().to_dict() base["sandbox_name"] = self.sandbox_name diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 624f66b01..91a26c7cf 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -104,8 +104,10 @@ "the judge couldn't decide any check — check its setup" when every check is undecided. - ``run_step_id``, ``run_kind``, ``user_sim``, ``judge_model``; ``instructions`` (``source``, ``prompt_sha256``, ``recorded_prompt_sha256``, ``matches``); ``rubric_persona`` (when ``rubric_persona_sha256`` is set); ``notes`` - (what the judge was told about gaps in the record); ``graded_turns``, ``turns``; ``calls``, ``usage``, ``cost``; and - rubrics_verifier's ``judge_output_retries`` / ``judge_output_discrepancies`` / ``judge_batches``. + (what the judge was told about gaps in the record); ``graded_turns``, ``turns``; ``calls``, ``usage``, ``cost`` (the + calls' costs summed, each what the endpoint billed, else LiteLLM's price map's figure; None when no call's has one) + and ``unpriced_calls`` (the calls whose cost has no figure, when there are some); and rubrics_verifier's + ``judge_output_retries`` / ``judge_output_discrepancies`` / ``judge_batches``. - ``skipped`` instead, with ``score`` None and no rows, for a run that isn't judged: one that ended in an error (the always_on record's or the prompt response's ``error_type``: every verifier skips its prompt anyway), a chat run whose user sim never read, and a run with no graded turn. @@ -148,6 +150,7 @@ from dataclasses import dataclass, field from typing import Any, ClassVar, Optional +from agent_env.config import model_call from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core @@ -205,7 +208,8 @@ class _ThisRun: first_error: Optional[BaseException] = None # what the first call raised, if it did calls: int = 0 usage: list[Any] = field(default_factory=list) - cost: float = 0.0 + cost: float = 0.0 # the calls' costs that have a figure + unpriced: int = 0 # the calls whose cost has none class UserSimJudge(RubricsVerifierTaskStep): @@ -383,7 +387,7 @@ def _finish(self, judged: RubricsVerdict, entry: dict, this: _ThisRun) -> Verifi else: error["fails_task"] = True output = {**judged.output, **entry, "user_sim_error": error, "calls": this.calls, - "usage": core.usage_total(this.usage), "cost": round(this.cost, COST_DECIMALS)} + "usage": core.usage_total(this.usage), **_cost(this)} if error["value"] == "unknown": checks = [r for r in rows if r["id"] in core.CHECKS] output["flag"] = FLAG_NONE_DECIDED if checks and all(r["verdict"] == "error" for r in checks) \ @@ -447,7 +451,10 @@ async def call() -> dict: cached_prefix=head) this.calls += 1 this.usage.append(got.get("usage")) - this.cost += got.get("cost") or 0.0 + if got.get("cost_usd") is None: + this.unpriced += 1 + else: + this.cost += got["cost_usd"] return got if this.first is None: @@ -500,22 +507,22 @@ def _root(e: BaseException) -> BaseException: def _setup_error(e: BaseException) -> Optional[BaseException]: - """The setup error behind a failed judge call, or None for a flaky one. Setup: agent-env's ConfigError (no key, - missing config) and LiteLLM's AuthenticationError, PermissionDeniedError, NotFoundError (an unknown model), - BadRequestError (with its kinds: an unknown provider, unsupported params, a prompt over the context window), - UnprocessableEntityError and BudgetExceededError. Anything else is flaky (Timeout, RateLimitError, - ServiceUnavailableError, InternalServerError, BadGatewayError, APIConnectionError, an agent judge's TimeoutError, - JudgeTimeoutError or AgentUnreachableError, an answer still unusable after its retries) and leaves the call's rows - undecided.""" - import litellm - - from agent_env.config.errors import ConfigError - - setup = (ConfigError, litellm.AuthenticationError, litellm.PermissionDeniedError, litellm.NotFoundError, - litellm.BadRequestError, litellm.UnprocessableEntityError, litellm.BudgetExceededError) + """The setup error behind a failed judge call (``model_call.setup_errors``), or None for a flaky one. Anything + else is flaky (Timeout, RateLimitError, ServiceUnavailableError, InternalServerError, BadGatewayError, + APIConnectionError, an agent judge's TimeoutError, JudgeTimeoutError or AgentUnreachableError, an answer still + unusable after its retries) and leaves the call's rows undecided.""" + setup = model_call.setup_errors() return next((x for x in _chain(e) if isinstance(x, setup)), None) +def _cost(this: _ThisRun) -> dict[str, Any]: + """The run's ``cost``: its calls' costs summed, None when no call's cost has a figure, and ``unpriced_calls``, the + calls left out of the sum, when there are some.""" + priced = this.calls - this.unpriced + total = round(this.cost, COST_DECIMALS) if priced or not this.calls else None + return {"cost": total, **({"unpriced_calls": this.unpriced} if this.unpriced else {})} + + def _problems(model: Any, items: Any, gate: Any, rubric_persona: Any = None) -> list[str]: out = [] if rubric_persona is not None and not (isinstance(rubric_persona, str) diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index 71059dfd1..840b1620c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -161,6 +161,8 @@ def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: def config_problems(self) -> list[str]: problems = super().config_problems() + if not isinstance(self.verifier_id, str): + problems.append(f"verifier_id must be a string, not {type(self.verifier_id).__name__}") if not 0.0 < self.min_passing_score <= 1.0: problems.append(f"min_passing_score must be above 0 and at most 1, not {self.min_passing_score}") return problems diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index d73ae15e3..e25333904 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -26,7 +26,7 @@ import re import shlex import uuid -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional from agent_env.providers.sandbox_providers.sandbox import VmSandbox from agent_env.providers.sandbox_providers.sandbox_provider import ( @@ -49,10 +49,36 @@ "probe_file_contains", "bash_cmd_succeeds", } +_PROBES = ("probe_file_exists", "probe_dir_exists", "probe_file_contains") _DEFAULT_BASE_DIR = "/app" _DEFAULT_SHELL_TIMEOUT_S = 120 +def _criterion_problems(criterion: Any) -> list[str]: + if not isinstance(criterion, dict): + return [f"must be an object, not {type(criterion).__name__}"] + rtype = criterion.get("type") + if not isinstance(rtype, str): + return [f"type must be a string, not {rtype!r}"] + problems = [] + paths = criterion.get("paths") + if rtype in _PROBES and not (isinstance(paths, list) and paths and all(isinstance(path, str) for path in paths)): + problems.append(f"{rtype} needs paths, a non-empty list of strings, not {paths!r}") + if rtype == "probe_file_contains": + pattern = criterion.get("expected") or criterion.get("value") or "" + if not isinstance(pattern, str): + problems.append(f"expected must be a string, not {pattern!r}") + else: + try: + re.compile(pattern) + except re.error as e: + problems.append(f"expected {pattern!r} doesn't compile as a regex: {e}") + command = criterion.get("bash_cmd") + if rtype == "bash_cmd_succeeds" and not (isinstance(command, str) and command.strip()): + problems.append(f"bash_cmd_succeeds needs a bash_cmd, a non-empty string, not {command!r}") + return problems + + def _outcome(passed: bool, justification: str) -> dict: return {"score": 1.0 if passed else 0.0, "passed": passed, "justification": justification} @@ -103,6 +129,30 @@ def to_dict(self) -> dict: base["verifier_id"] = self.verifier_id return base + def config_problems(self) -> list[str]: + """The criteria a run can't read, or that fail their row whatever the sandbox holds: one that isn't an object or + whose type isn't a string, a probe without a list of paths, a ``probe_file_contains`` pattern that doesn't + compile, a ``bash_cmd_succeeds`` with no command.""" + problems = super().config_problems() + for idx, criterion in enumerate(self.criteria): + problems.extend(f"criterion #{idx}: {problem}" for problem in _criterion_problems(criterion)) + return problems + + def config_warnings(self) -> list[str]: + """The criteria a run grades as nothing: a type this step doesn't handle, which it skips, and a + ``probe_file_contains`` with no expected text, which any file passes.""" + warnings = super().config_warnings() + for idx, criterion in enumerate(self.criteria): + if not (isinstance(criterion, dict) and isinstance(criterion.get("type"), str)): + continue # config_problems refuses it + rtype = criterion["type"] + if rtype not in _HANDLED_TYPES: + warnings.append(f"criterion #{idx}: type {rtype!r} isn't one verify_sandbox checks, so a run skips " + f"it; it checks {sorted(_HANDLED_TYPES)}") + elif rtype == "probe_file_contains" and not (criterion.get("expected") or criterion.get("value")): + warnings.append(f"criterion #{idx}: probe_file_contains has no expected text, so any file passes") + return warnings + @classmethod def from_dict(cls, data: dict) -> "VerifySandboxTaskStep": return cls( diff --git a/src/agent_env/training/harness.py b/src/agent_env/training/harness.py index 7f09217ac..aecd0754f 100644 --- a/src/agent_env/training/harness.py +++ b/src/agent_env/training/harness.py @@ -50,13 +50,13 @@ def prompt_rows(bundle: str | Path, *, tasks: Sequence[str] = (), evals: Sequence[str] = (), - agent: str | None = None) -> list[dict[str, Any]]: - """The trainer's dataset, a row per task ``AgentEnvSessionFactory`` plays with the same arguments: a system line - naming the task, which is how the factory finds it, then the task's first prompt, for TRL's logs. The model never - sees this prompt: the task's own agent builds its own.""" + agent: str | None = None, skip_lint: bool = False) -> list[dict[str, Any]]: + """The trainer's dataset, a row per task ``AgentEnvSessionFactory`` plays with the same arguments, ``skip_lint`` + among them: a system line naming the task, which is how the factory finds it, then the task's first prompt, for + TRL's logs. The model never sees this prompt: the task's own agent builds its own.""" return [{"prompt": [{"role": "system", "content": f"AgentEnv task {index}: {task.id}"}, {"role": "user", "content": _first_prompt(task)}], "task_id": task.id} - for index, (task, _) in enumerate(_trained(bundle, tasks, evals, agent))] + for index, (task, _) in enumerate(_trained(bundle, tasks, evals, agent, skip_lint))] class AgentEnvSessionFactory(ResourceSessionFactory): @@ -64,14 +64,16 @@ class AgentEnvSessionFactory(ResourceSessionFactory): bundle to run on local Docker, checks that ``llm_url``'s engine returns token ids, and serves OpenEnv's capture proxy in front of it, where the task's solver reaches it: the agent a step prompts, which it deploys, its other agents and judges keeping the configured endpoint. TRL sets ``model`` to the policy it publishes; ``reward`` is as - ``build_app``'s, and ``agent`` the agent the solver deploys, as ``agent-env run --agent``'s.""" + ``build_app``'s, ``agent`` the agent the solver deploys, as ``agent-env run --agent``'s, and ``skip_lint`` as + ``agent-env run --skip-lint``'s.""" def __init__(self, bundle: str | Path, *, llm_url: str, model: str, sampling: dict[str, Any] | None = None, tasks: Sequence[str] = (), evals: Sequence[str] = (), - reward: str | Callable[[dict[str, float]], float] | None = None, agent: str | None = None): + reward: str | Callable[[dict[str, float]], float] | None = None, agent: str | None = None, + skip_lint: bool = False): self.model = model self._llm_url, self._sampling = llm_url, sampling - self._trained = _trained(bundle, tasks, evals, agent) + self._trained = _trained(bundle, tasks, evals, agent, skip_lint) self._rewards = [_reward(task, reward) for task, _ in self._trained] require_llm(llm_url, model, require_tokens=True) self._proxy = CaptureServer(llm_url=llm_url, model=model, port=_free_port(), @@ -178,11 +180,11 @@ async def _run(self, timeout: float | None) -> int: self._done.set() -def _trained(bundle: str | Path, tasks: Sequence[str], evals: Sequence[str], - agent: str | None) -> list[tuple[Task, dict[str, dict[str, Any]]]]: +def _trained(bundle: str | Path, tasks: Sequence[str], evals: Sequence[str], agent: str | None, + skip_lint: bool = False) -> list[tuple[Task, dict[str, dict[str, Any]]]]: """The selected tasks that can train their solver, prepared as ``agent-env run`` prepares them, each with its solver's deploys and their step params, ``agent``'s if it swaps the solver. The others are left out before they're - checked, so what they alone need can't stop the rest.""" + checked, task lint included, so what they alone need can't stop the rest.""" with namespace_routing(): kept = [] for entry in planned_tasks(bundle, tasks=tasks, evals=evals, agent=agent): @@ -193,7 +195,8 @@ def _trained(bundle: str | Path, tasks: Sequence[str], evals: Sequence[str], if not kept: raise ValueError("no task of the bundle has a solver to train: an agent a step prompts, deployed by a " "deploy_agent step, that isn't also its judge") - prepared = prepare_bundle(bundle, tasks=kept, sandbox=_SANDBOX, agent=agent, solver_endpoint=True) + prepared = prepare_bundle(bundle, tasks=kept, sandbox=_SANDBOX, agent=agent, solver_endpoint=True, + skip_lint=skip_lint) swapped = solver_params(prepared.plan, prepared.materialization) return [(task, {step: swapped.get(entry.id, {}).get(step, {}) for step in solver_steps([step.to_dict() for step in task.steps])}) diff --git a/src/agent_env/training/nemo_gym/__main__.py b/src/agent_env/training/nemo_gym/__main__.py index f426bddea..812b61edc 100644 --- a/src/agent_env/training/nemo_gym/__main__.py +++ b/src/agent_env/training/nemo_gym/__main__.py @@ -38,13 +38,15 @@ def main() -> None: "reward is then their sum.") @click.option("--max-concurrent-envs", type=int, default=8, show_default=True, help="The episodes the server runs at once, which should cover a step's rollouts: the rest wait their turn.") +@click.option("--skip-lint", is_flag=True, + help="Serve tasks agent-env task lint refuses for its own rules, as agent-env run --skip-lint.") def write(source: str, root: str, name: str | None, sandbox: str | None, reward: str | None, reward_components: bool, - max_concurrent_envs: int) -> None: + max_concurrent_envs: int, skip_lint: bool) -> None: """Writes the resources server for SOURCE, a bundle or the URL of an OpenEnv server serving one, into the Gym root ROOT, with its config and a JSONL file of rows per split. Playing one task of each toolset to find its tools deploys its envs.""" server = write_server(source, root, name=name, sandbox=sandbox, reward=reward, reward_components=reward_components, - max_concurrent_envs=max_concurrent_envs) + max_concurrent_envs=max_concurrent_envs, skip_lint=skip_lint) click.echo(f"wrote {server}") diff --git a/src/agent_env/training/nemo_gym/prepare.py b/src/agent_env/training/nemo_gym/prepare.py index 2b8f404f4..18eef47e3 100644 --- a/src/agent_env/training/nemo_gym/prepare.py +++ b/src/agent_env/training/nemo_gym/prepare.py @@ -48,7 +48,7 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = None, sandbox: str | None = None, reward: str | None = None, reward_components: bool = False, max_concurrent_envs: int = 8, - reset_timeout: float = 600.0) -> Path: + reset_timeout: float = 600.0, skip_lint: bool = False) -> Path: """``source``'s tasks as NeMo Gym's resources server ``name``, the bundle's env name by default, under the Gym root ``root``, returning the server's folder ``resources_servers/``. It holds the server's ``app.py`` and ``pyproject.toml``; ``configs/.yaml``, which runs the server, ``simple_agent`` to play its rows and the @@ -58,14 +58,15 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = Non ``source`` is the URL of an OpenEnv server serving the bundle, which the config names, or a bundle, a folder or an installed bundle's name, which the resources server serves itself, deploying where ``sandbox`` names, at most ``max_concurrent_envs`` episodes at once. ``reward_components`` has the server answer verify with each of a task's - scores, for NeMo RL's GDPO, their sum the reward, so it takes no ``reward``. A row's tools are every tool its - task's turns offer, found by playing one task of each toolset to its last turn, unscored, so writing deploys - those.""" + scores, for NeMo RL's GDPO, their sum the reward, so it takes no ``reward``. ``skip_lint`` serves tasks task + lint's own rules refuse, as ``build_app``'s, and the config the server reads keeps it. A row's tools are every + tool its task's turns offer, found by playing one task of each toolset to its last turn, unscored, so writing + deploys those.""" if reward is not None and reward_components: raise ValueError(REWARD_AND_COMPONENTS) url = str(source) if str(source).startswith(("http://", "https://")) else None served = None if url else Server(build_app(source, reward=summed if reward_components else reward, sandbox=sandbox, - max_concurrent_envs=1)) + max_concurrent_envs=1, skip_lint=skip_lint)) try: env_name, tasks, probed = tasks_and_tools(url or served.url, local="name the bundle's folder instead of its " "server", timeout=reset_timeout) @@ -83,6 +84,8 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = Non **({"sandbox": sandbox} if sandbox else {}), **({"reward": reward} if reward else {})} if reward_components: where["reward_components"] = True + if skip_lint and not url: + where["skip_lint"] = True server = Path(root) / "resources_servers" / name (server / "configs").mkdir(parents=True, exist_ok=True) (server / "data").mkdir(exist_ok=True) diff --git a/src/agent_env/training/nemo_gym/server.py b/src/agent_env/training/nemo_gym/server.py index e245675fc..dec1206d2 100644 --- a/src/agent_env/training/nemo_gym/server.py +++ b/src/agent_env/training/nemo_gym/server.py @@ -62,7 +62,8 @@ class AgentEnvResourcesServerConfig(BaseResourcesServerConfig): as NeMo RL's GDPO takes them, and makes the reward their sum, as NeMo RL requires of a reward with components; it takes no ``reward``. ``masked_as_error`` answers verify for a masked rollout with HTTP 503 and why, rather than with ``mask_sample``, for a client that doesn't read ``mask_sample``: verifiers' NeMo Gym taskset then fails the - rollout, which prime-rl leaves out of the loss and its group's baseline. It runs with one worker, since a rollout's + rollout, which prime-rl leaves out of the loss and its group's baseline. ``skip_lint`` serves a bundle's tasks task + lint's own rules refuse, as ``build_app``'s. It runs with one worker, since a rollout's session lives in its process.""" url: str | None = None @@ -75,6 +76,7 @@ class AgentEnvResourcesServerConfig(BaseResourcesServerConfig): idle_timeout: float = Field(600.0, gt=0, allow_inf_nan=False) reward_components: bool = False masked_as_error: bool = False + skip_lint: bool = False @model_validator(mode="after") def _checked(self) -> "AgentEnvResourcesServerConfig": @@ -198,7 +200,7 @@ def _url(self) -> str: def _serve(self) -> Server: reward = summed if self.config.reward_components else self.config.reward return Server(build_app(self.config.bundle, reward=reward, sandbox=self.config.sandbox, - max_concurrent_envs=self.config.max_concurrent_envs)) + max_concurrent_envs=self.config.max_concurrent_envs, skip_lint=self.config.skip_lint)) async def _shut_down(self) -> None: """Ends the episodes left running, then stops the server this one served the bundle on, which tears down its diff --git a/src/agent_env/training/trl.py b/src/agent_env/training/trl.py index fc1de3f8b..87c451695 100644 --- a/src/agent_env/training/trl.py +++ b/src/agent_env/training/trl.py @@ -103,11 +103,13 @@ def __init__(self, url: str, split: str = "all", *, tool_timeout: float = 60.0, @classmethod def local(cls, bundle: str | Path | Mapping[str, str | Path], split: str = "all", *, reward: str | Callable[[dict[str, float]], float] | None = None, sandbox: str | None = None, - max_concurrent_envs: int = 8, tool_timeout: float = 60.0, reset_timeout: float = 600.0) -> AgentEnv: + max_concurrent_envs: int = 8, tool_timeout: float = 60.0, reset_timeout: float = 600.0, + skip_lint: bool = False) -> AgentEnv: """``bundle`` served by ``build_app`` in this process on a loopback port, for a trainer on a machine that can - deploy its envs. ``max_concurrent_envs`` must cover the rollouts the trainer generates at once. The server - stops at ``close``, or when the process exits, ending its episodes.""" - server = Server(build_app(bundle, reward=reward, sandbox=sandbox, max_concurrent_envs=max_concurrent_envs)) + deploy its envs. ``max_concurrent_envs`` must cover the rollouts the trainer generates at once; ``skip_lint`` is + ``build_app``'s. The server stops at ``close``, or when the process exits, ending its episodes.""" + server = Server(build_app(bundle, reward=reward, sandbox=sandbox, max_concurrent_envs=max_concurrent_envs, + skip_lint=skip_lint)) _servers.add(server) try: env = cls(server.url, split, tool_timeout=tool_timeout, reset_timeout=reset_timeout) diff --git a/tst/data/verifier_golden/judge_inputs.json b/tst/data/verifier_golden/judge_inputs.json new file mode 100644 index 000000000..51d69d89a --- /dev/null +++ b/tst/data/verifier_golden/judge_inputs.json @@ -0,0 +1,1818 @@ +{ + "model_without_the_trajectory": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "ff57202774b5d1abd17f0e48a493cde8142596e8b67d97b5c1bb8dc474006238", + "chars": 1962 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_one_turn": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "fb0826457f7e4bb338d54be62328d5ff01518fb640cdd10dff53d70b5802afe6", + "chars": 48486 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_one_turn_from_the_per_turn_urls": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "fb0826457f7e4bb338d54be62328d5ff01518fb640cdd10dff53d70b5802afe6", + "chars": 48486 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_one_turn_compacted": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "bcab6d9aee4c77239aad769ee563927516676572e91650097b55aae78cda0f65", + "chars": 3118 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_two_turns": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "ef3944b33d70fcbdd59ad31e1ef5779544162db33bae5192adf37b1109c5a111", + "chars": 72237 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_two_turns_compacted": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "287d778013022f537d734be65a003532a2b6b93a47ed60e3ffeba9acf2f09354", + "chars": 3698 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_graded_up_to_turn_one": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "9c2e6c7bbad7d96e3665a8b5896221160f25f2600ce52c702c78a7852453f778", + "chars": 25839 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "model_screenshots_per_criterion": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "d671aef53960b3af8ea954f97b8490d33d087a286b670192319c2688475bfd81", + "chars": 5073 + }, + "images": [ + { + "type": "text", + "text": "FRAME: action 1" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + }, + { + "type": "text", + "text": "FRAME: action 3" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + }, + { + "type": "text", + "text": "FRAME: action 6" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + }, + { + "type": "text", + "text": "FRAME: final action 9" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + }, + { + "type": "text", + "text": "FRAME: final action 10" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + }, + { + "type": "text", + "text": "FRAME: final action 11" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + } + ], + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "agent_one_turn": { + "calls": [ + { + "prompt": { + "sha256": "b0fb053a82999d1f59eb6d8e92eeff64258f9d304735575c79204923cfee3016", + "chars": 2080 + } + } + ], + "files": { + "/tmp/prompt_trajectory.json": { + "text": { + "sha256": "97bafff131caea63b3c514b783bcc1267d8c683f8646ad3f63feed652f218a52", + "chars": 46433 + } + } + } + }, + "agent_one_turn_compacted": { + "calls": [ + { + "prompt": { + "sha256": "b0fb053a82999d1f59eb6d8e92eeff64258f9d304735575c79204923cfee3016", + "chars": 2080 + } + } + ], + "files": { + "/tmp/prompt_trajectory.json": { + "text": { + "sha256": "e34a69039c30b0b6444ca199fd1ec70b1b90e7813b649e13c3786feb847b8008", + "chars": 1065 + } + }, + "/tmp/tool_call_results/tool_call_result_1.json": { + "text": { + "sha256": "d3fed826fcf45e66b89354b7feba889b96a30c64183f45c42caa02b5f4e159a4", + "chars": 3202 + } + }, + "/tmp/tool_call_results/tool_call_result_2.json": { + "text": { + "sha256": "c29f1442846a2c41ed1da818e6df89778ad39352593e88277f37953e528b10bc", + "chars": 3202 + } + }, + "/tmp/tool_call_results/tool_call_result_3.json": { + "text": { + "sha256": "2e76ad6ab2ba2892b17b9b01b3376c3bda382780b301f4f8c504aaf50e50365f", + "chars": 3202 + } + } + } + }, + "agent_two_turns": { + "calls": [ + { + "prompt": { + "sha256": "bdae7f5b3d511c1e8adca39c9cbb22bc30c74f937a22c9d45ae46c289f07bfe9", + "chars": 2384 + } + } + ], + "files": { + "/tmp/prompt_trajectory_turns//turn_01.json": { + "text": { + "sha256": "919e68cb5e5ff44462a474b2466b1d6fd44022d980db4568581f601b36994b6d", + "chars": 23711 + } + }, + "/tmp/prompt_trajectory_turns//turn_02.json": { + "text": { + "sha256": "97bafff131caea63b3c514b783bcc1267d8c683f8646ad3f63feed652f218a52", + "chars": 46433 + } + } + } + }, + "agent_two_turns_compacted": { + "calls": [ + { + "prompt": { + "sha256": "bdae7f5b3d511c1e8adca39c9cbb22bc30c74f937a22c9d45ae46c289f07bfe9", + "chars": 2384 + } + } + ], + "files": { + "/tmp/prompt_trajectory_turns//turn_01.json": { + "text": { + "sha256": "a5fca5a9f6b5bc0f96cc572cbbd4f3e5894a4b5353dda57a6da87aaaa7080b39", + "chars": 516 + } + }, + "/tmp/prompt_trajectory_turns//turn_02.json": { + "text": { + "sha256": "0ca88f388b2ffa752e8cba1898c42348600064e1092f720777bb347bd8e1df58", + "chars": 1089 + } + }, + "/tmp/tool_call_results/turn_01_tool_call_result_1.json": { + "text": { + "sha256": "d3fed826fcf45e66b89354b7feba889b96a30c64183f45c42caa02b5f4e159a4", + "chars": 3202 + } + }, + "/tmp/tool_call_results/turn_02_tool_call_result_1.json": { + "text": { + "sha256": "d3fed826fcf45e66b89354b7feba889b96a30c64183f45c42caa02b5f4e159a4", + "chars": 3202 + } + }, + "/tmp/tool_call_results/turn_02_tool_call_result_2.json": { + "text": { + "sha256": "c29f1442846a2c41ed1da818e6df89778ad39352593e88277f37953e528b10bc", + "chars": 3202 + } + }, + "/tmp/tool_call_results/turn_02_tool_call_result_3.json": { + "text": { + "sha256": "2e76ad6ab2ba2892b17b9b01b3376c3bda382780b301f4f8c504aaf50e50365f", + "chars": 3202 + } + } + } + }, + "always_on_unstaged": { + "prompt": { + "sha256": "07e111aea82357adc2b46c70ca9dbfbf67a5d5ce7ef0662b3771f9e677e2d151", + "chars": 3742 + } + }, + "always_on_staged": { + "prompt": { + "sha256": "b9f74aebb0a033e2b7a24b5c35030bb7a3ecdd3c23e2f57643667e3b7dc86c12", + "chars": 11552 + }, + "files": { + "/tmp/always_on_run//events.jsonl": { + "sha256": "f2ebab0ebc8b5960699b6401c47356fdb05f9da48835bf35482e8ca5ac061401", + "chars": 29079 + }, + "/tmp/always_on_run//index.md": { + "sha256": "8a1f8a5ef9765fda6a418236a7d69bad8823a55e19ba67484f06e546fff46465", + "chars": 5044 + }, + "/tmp/always_on_run//triggers.jsonl": "", + "/tmp/always_on_run//wakes/results/wake_002_call_001_gmail_get_thread.json": { + "sha256": "3e0a1cd9d8f2dad998b721bab9947b3e3fae02a37c01b129c831b7640d5c9e7f", + "chars": 5287 + }, + "/tmp/always_on_run//wakes/results/wake_002_call_004_gcal_patch_event.json": { + "sha256": "4bf38e06c941d545c4f008fada4089ada1eef38eb9fb27d8b737a9ea4da76195", + "chars": 1894 + }, + "/tmp/always_on_run//wakes/results/wake_002_call_005_gcal_create_event.json": { + "sha256": "09c00b235ca997cb9e13f5bd56e3e80492a3af0ccb24a88333c313fe74ed1920", + "chars": 1633 + }, + "/tmp/always_on_run//wakes/wake_001.jsonl": { + "sha256": "6b470cc637e6af72d036f6b814a0c7b881a91e227da8d59ebd37eaa6d2aeff04", + "chars": 5654 + }, + "/tmp/always_on_run//wakes/wake_002.jsonl": { + "sha256": "eb54ef3045061c83be24ae4a5f70122d4e5da3abefefa22dba036a5b477adaad", + "chars": 13184 + }, + "/tmp/always_on_run//wakes/wake_003.jsonl": { + "sha256": "25f3022c96a010b21d2e50f9ce15341e5fc0f4fb88a5e10ec8f42a33e8e76141", + "chars": 4890 + }, + "/tmp/always_on_run//wakes/wake_004.jsonl": { + "sha256": "19afb8bc30fb6b487712fa44601a0471571e4eb0b48f1bfe4ac79a439ebf0f84", + "chars": 4831 + } + } + }, + "user_sim": [ + { + "model": "fake/judge", + "names": [ + "delegation" + ], + "criteria": [ + { + "id": "c3", + "check": "Delegation (A4)", + "question": { + "sha256": "07a74a517c5ede32884839749db2046b29326d16aba8bb051e3fd707a8f8bd1c", + "chars": 1181 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "0b16063f48d9ed66433bd55a8eff296b37775d4c05de43943e89792ea92c7afd", + "chars": 15573 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + }, + { + "model": "fake/judge", + "names": [ + "ending" + ], + "criteria": [ + { + "id": "c7", + "check": "Ending (G4)", + "question": { + "sha256": "54238cbf0af0a835abdfe2d0cbf334aa3e873da78c2f9c31a2295b1e9fbb9989", + "chars": 569 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "7192bbd21ddad67614034dd1e01687c126e3a5f4009c20ae9b205376323ad6c0", + "chars": 14949 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + }, + { + "model": "fake/judge", + "names": [ + "guidance" + ], + "criteria": [ + { + "id": "c1", + "check": "Guidance (A1, A3)", + "question": { + "sha256": "9a9ea029a613397095f89de8b8f3aa230d8e22acf30a8c070f7259452fa6885d", + "chars": 2301 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "16eb28a8fa8b7bf10de918004e1c3a1f575a5e4ee01c94d263ebfd86215e88c9", + "chars": 16703 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + }, + { + "model": "fake/judge", + "names": [ + "made_up" + ], + "criteria": [ + { + "id": "c6", + "check": "Made-up facts (A2)", + "question": { + "sha256": "0481faff49e323c5554f335302e31b91cbbd214ff6212d480450d22a94a55f31", + "chars": 1153 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "da12e6ea6905a5759285955e9279fea51b3564129a7f4ecfda713bd1ea06bfce", + "chars": 15549 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + }, + { + "model": "fake/judge", + "names": [ + "oversharing" + ], + "criteria": [ + { + "id": "c2", + "check": "Oversharing (G1, G2)", + "question": { + "sha256": "5d10ce962a3dd5c76c82a573361afae18e961e70a70f7e476a79336262a240f3", + "chars": 1675 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "3f1f9ba1b23d0efbb89cc26ac52820f9f65dbe01f26239cd9143c29dea819b22", + "chars": 16069 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + }, + { + "model": "fake/judge", + "names": [ + "rogue" + ], + "criteria": [ + { + "id": "c5", + "check": "Rogue (G3, G5)", + "question": { + "sha256": "b1b197d9df1e6549cbaa0bcde06a8fccc9b7f32f8671cc8cc01f5b855ff89e5f", + "chars": 1216 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "dca98efe567efed02898cb1d52600a91926f134a9fcdd8c6be5f8e98663b71b1", + "chars": 15605 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + }, + { + "model": "fake/judge", + "names": [ + "style" + ], + "criteria": [ + { + "id": "c4", + "check": "Style (F1, F2, F3, F4)", + "question": { + "sha256": "fb856004c2a0f4f8c686370131a8672adfe0702967a7e4aceb27deb5b565bee9", + "chars": 1530 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 0.0 + } + ], + "prompt": { + "sha256": "876cccf5b30b3733ac6f4643383ac6bdab6cc5270a448b4ad37c209c766ff5c7", + "chars": 15919 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "fdbc93c464b4fa4c9d94858ae17651667badd777f83fa8540d026bb3ded7d719", + "chars": 14079 + } + } + ], + "scripted_verify": { + "url": "http://mcp", + "trajectory": [ + null, + [ + { + "type": "system", + "subtype": "init", + "cwd": "/app", + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "tools": [ + "mcp__apps__browser_click", + "mcp__apps__browser_close", + "mcp__apps__browser_drag", + "mcp__apps__browser_fill_form", + "mcp__apps__browser_handle_dialog", + "mcp__apps__browser_hover", + "mcp__apps__browser_navigate", + "mcp__apps__browser_navigate_back", + "mcp__apps__browser_press_key", + "mcp__apps__browser_resize", + "mcp__apps__browser_select_option", + "mcp__apps__browser_snapshot" + ], + "mcp_servers": [ + { + "name": "apps", + "status": "connected" + } + ], + "model": "claude-opus-5-5", + "permissionMode": "bypassPermissions", + "slash_commands": [ + "deep-research", + "design-sync", + "update-config", + "verify", + "debug", + "code-review", + "simplify", + "batch", + "fewer-permission-prompts", + "loop", + "claude-api", + "run", + "run-skill-generator", + "clear", + "compact", + "context", + "heapdump", + "init", + "reload-skills", + "review", + "security-review", + "usage", + "insights", + "goal", + "team-onboarding" + ], + "apiKeySource": "none", + "claude_code_version": "2.1.177", + "output_style": "default", + "agents": [ + "claude", + "Explore", + "general-purpose", + "Plan", + "statusline-setup" + ], + "skills": [ + "deep-research", + "design-sync", + "update-config", + "verify", + "debug", + "code-review", + "simplify", + "batch", + "fewer-permission-prompts", + "loop", + "claude-api", + "run", + "run-skill-generator" + ], + "plugins": [], + "analytics_disabled": false, + "product_feedback_disabled": false, + "uuid": "ae6a3cf3-999c-4907-8283-7db48f3c7c41", + "memory_paths": { + "auto": "/root/.claude/projects/-app/memory/" + }, + "fast_mode_state": "off" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ27di7jDQ2FrfnSaqQu", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_014S2or6YhfoaiEFUJU21h4E", + "name": "mcp__apps__gmail_get_thread", + "input": { + "user_email": "tess.marlowe@example.test", + "thread_id": "nw-1187" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 3613, + "cache_read_input_tokens": 55808, + "cache_creation": { + "ephemeral_5m_input_tokens": 3613, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 8 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "cc813fa8-840b-4c22-ae09-7e9b553ac8f3" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_014S2or6YhfoaiEFUJU21h4E", + "type": "tool_result", + "content": { + "sha256": "18baeda3e6122970719ef82eb0ea05c0bb18163b7607886f3629a16f0fe13a49", + "chars": 5127 + } + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "ce3d07b7-8976-4063-9752-852eee77ba2c", + "timestamp": "2026-10-30T20:03:32.476Z", + "tool_use_result": { + "content": { + "sha256": "18baeda3e6122970719ef82eb0ea05c0bb18163b7607886f3629a16f0fe13a49", + "chars": 5127 + }, + "structuredContent": { + "result": { + "sha256": "b54f1e197b9abe7f6c56cc134e5149fe77bd066884c9e568139ed8d22139e004", + "chars": 4694 + } + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3amiSMU2XRHLsZ5hE8", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "", + "signature": "CAQStgUKEggTEAIYAjgBQgh0aGlua2luZxIM5FH92PjkGiAY1XucGgzURGHQaHkLkApdkq4iMJZBlxvXqmfzZvbrIcoaHI9XuRyTOR0zM7cdjOgSLrqWNlI/" + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 2036, + "cache_read_input_tokens": 465148, + "cache_creation": { + "ephemeral_5m_input_tokens": 2036, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 4 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "411de34e-c6ac-492f-86dc-99f0b0483cb6" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3amiSMU2XRHLsZ5hE8", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01KhowokbuAvM2xyD8EwpXH9", + "name": "mcp__apps__gmail_list_messages", + "input": { + "user_email": "tess.marlowe@example.test", + "query": "from:packright.example.test newer_than:2d" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 2036, + "cache_read_input_tokens": 465148, + "cache_creation": { + "ephemeral_5m_input_tokens": 2036, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 4 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "103b2889-3b01-43a9-966a-46dc28d3be45" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01KhowokbuAvM2xyD8EwpXH9", + "type": "tool_result", + "content": "{\"result\":\"{\\\"messages\\\": [], \\\"resultSizeEstimate\\\": 0, \\\"nextPageToken\\\": null}\"}" + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "f0b36a3b-49cd-48b2-a329-58f671a4f73b", + "timestamp": "2026-10-30T20:22:48.018Z", + "tool_use_result": { + "content": "{\"result\":\"{\\\"messages\\\": [], \\\"resultSizeEstimate\\\": 0, \\\"nextPageToken\\\": null}\"}", + "structuredContent": { + "result": "{\"messages\": [], \"resultSizeEstimate\": 0, \"nextPageToken\": null}" + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "", + "signature": "CAQS3wYKEggTEAIYAjgBQgh0aGlua2luZxIMa2rBO+8KWmCdvVF4GgzyV/1KKvqQeg9REUciMMkbAyQV+klA/oGe8KHFfdllylKG6ClKyX27M/VmuvU3aEMV" + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "b920b9d3-d877-4751-b211-9c3686f86f17" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Still nothing from PackRight. I'll note the contingency on the plan and the calendar so it doesn't get lost:" + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "b81fbbf6-0d84-4c09-bd31-ce33af50baf2" + } + ], + [ + { + "type": "system", + "subtype": "init", + "cwd": "/app", + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "tools": [ + "mcp__apps__browser_click", + "mcp__apps__browser_close", + "mcp__apps__browser_drag", + "mcp__apps__browser_fill_form", + "mcp__apps__browser_handle_dialog", + "mcp__apps__browser_hover", + "mcp__apps__browser_navigate", + "mcp__apps__browser_navigate_back", + "mcp__apps__browser_press_key", + "mcp__apps__browser_resize", + "mcp__apps__browser_select_option", + "mcp__apps__browser_snapshot" + ], + "mcp_servers": [ + { + "name": "apps", + "status": "connected" + } + ], + "model": "claude-opus-5-5", + "permissionMode": "bypassPermissions", + "slash_commands": [ + "deep-research", + "design-sync", + "update-config", + "verify", + "debug", + "code-review", + "simplify", + "batch", + "fewer-permission-prompts", + "loop", + "claude-api", + "run", + "run-skill-generator", + "clear", + "compact", + "context", + "heapdump", + "init", + "reload-skills", + "review", + "security-review", + "usage", + "insights", + "goal", + "team-onboarding" + ], + "apiKeySource": "none", + "claude_code_version": "2.1.177", + "output_style": "default", + "agents": [ + "claude", + "Explore", + "general-purpose", + "Plan", + "statusline-setup" + ], + "skills": [ + "deep-research", + "design-sync", + "update-config", + "verify", + "debug", + "code-review", + "simplify", + "batch", + "fewer-permission-prompts", + "loop", + "claude-api", + "run", + "run-skill-generator" + ], + "plugins": [], + "analytics_disabled": false, + "product_feedback_disabled": false, + "uuid": "ae6a3cf3-999c-4907-8283-7db48f3c7c41", + "memory_paths": { + "auto": "/root/.claude/projects/-app/memory/" + }, + "fast_mode_state": "off" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ27di7jDQ2FrfnSaqQu", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_014S2or6YhfoaiEFUJU21h4E", + "name": "mcp__apps__gmail_get_thread", + "input": { + "user_email": "tess.marlowe@example.test", + "thread_id": "nw-1187" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 3613, + "cache_read_input_tokens": 55808, + "cache_creation": { + "ephemeral_5m_input_tokens": 3613, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 8 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "cc813fa8-840b-4c22-ae09-7e9b553ac8f3" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_014S2or6YhfoaiEFUJU21h4E", + "type": "tool_result", + "content": { + "sha256": "18baeda3e6122970719ef82eb0ea05c0bb18163b7607886f3629a16f0fe13a49", + "chars": 5127 + } + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "ce3d07b7-8976-4063-9752-852eee77ba2c", + "timestamp": "2026-10-30T20:03:32.476Z", + "tool_use_result": { + "content": { + "sha256": "18baeda3e6122970719ef82eb0ea05c0bb18163b7607886f3629a16f0fe13a49", + "chars": 5127 + }, + "structuredContent": { + "result": { + "sha256": "b54f1e197b9abe7f6c56cc134e5149fe77bd066884c9e568139ed8d22139e004", + "chars": 4694 + } + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3amiSMU2XRHLsZ5hE8", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "", + "signature": "CAQStgUKEggTEAIYAjgBQgh0aGlua2luZxIM5FH92PjkGiAY1XucGgzURGHQaHkLkApdkq4iMJZBlxvXqmfzZvbrIcoaHI9XuRyTOR0zM7cdjOgSLrqWNlI/" + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 2036, + "cache_read_input_tokens": 465148, + "cache_creation": { + "ephemeral_5m_input_tokens": 2036, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 4 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "411de34e-c6ac-492f-86dc-99f0b0483cb6" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3amiSMU2XRHLsZ5hE8", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01KhowokbuAvM2xyD8EwpXH9", + "name": "mcp__apps__gmail_list_messages", + "input": { + "user_email": "tess.marlowe@example.test", + "query": "from:packright.example.test newer_than:2d" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 4, + "cache_creation_input_tokens": 2036, + "cache_read_input_tokens": 465148, + "cache_creation": { + "ephemeral_5m_input_tokens": 2036, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 4 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "103b2889-3b01-43a9-966a-46dc28d3be45" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01KhowokbuAvM2xyD8EwpXH9", + "type": "tool_result", + "content": "{\"result\":\"{\\\"messages\\\": [], \\\"resultSizeEstimate\\\": 0, \\\"nextPageToken\\\": null}\"}" + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "f0b36a3b-49cd-48b2-a329-58f671a4f73b", + "timestamp": "2026-10-30T20:22:48.018Z", + "tool_use_result": { + "content": "{\"result\":\"{\\\"messages\\\": [], \\\"resultSizeEstimate\\\": 0, \\\"nextPageToken\\\": null}\"}", + "structuredContent": { + "result": "{\"messages\": [], \"resultSizeEstimate\": 0, \"nextPageToken\": null}" + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "", + "signature": "CAQS3wYKEggTEAIYAjgBQgh0aGlua2luZxIMa2rBO+8KWmCdvVF4GgzyV/1KKvqQeg9REUciMMkbAyQV+klA/oGe8KHFfdllylKG6ClKyX27M/VmuvU3aEMV" + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "b920b9d3-d877-4751-b211-9c3686f86f17" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Still nothing from PackRight. I'll note the contingency on the plan and the calendar so it doesn't get lost:" + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "b81fbbf6-0d84-4c09-bd31-ce33af50baf2" + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01GMe3D13HEyXzh9D8Nz7dN6", + "name": "mcp__apps__gsheets_values_update", + "input": { + "spreadsheetId": "18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ", + "range": "Pours!K5", + "valueInputOption": "RAW", + "values": [ + [ + "Slot held for Marcus: 40 custom-label Balsam tins, $16 each + label cost, waiting on PackRight. If the labels fall through, offer Rae ~40 tins (Balsam/Cranberry) at $9 by Dec 8 in this slot (Tess, Nov 1). Cures Dec 5; last pour ready for Riverside Dec 6." + ] + ] + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "a50ab956-9492-429a-9b47-020284dbfe09" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01GMe3D13HEyXzh9D8Nz7dN6", + "type": "tool_result", + "content": "{\"result\":\"{\\\"spreadsheetId\\\": \\\"18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ\\\", \\\"driveFileId\\\": \\\"18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ\\\", \\\"updatedRange\\\": \\\"Pours!K5:K5\\\", \\\"updatedRows\\\": 1, \\\"updatedColumns\\\": 1, \\\"updatedCells\\\": 1, \\\"modifiedTime\\\": \\\"2026-11-01T16:00:00Z\\\"}\"}" + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "50d2b0aa-cf93-4979-b85e-b0aaae5e54c9", + "timestamp": "2026-10-30T20:22:55.123Z", + "tool_use_result": { + "content": "{\"result\":\"{\\\"spreadsheetId\\\": \\\"18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ\\\", \\\"driveFileId\\\": \\\"18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ\\\", \\\"updatedRange\\\": \\\"Pours!K5:K5\\\", \\\"updatedRows\\\": 1, \\\"updatedColumns\\\": 1, \\\"updatedCells\\\": 1, \\\"modifiedTime\\\": \\\"2026-11-01T16:00:00Z\\\"}\"}", + "structuredContent": { + "result": "{\"spreadsheetId\": \"18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ\", \"driveFileId\": \"18Wi1exQPfM4W7K9b9qvNGwFDJwsTYg2zmwOrVSIQTzQ\", \"updatedRange\": \"Pours!K5:K5\", \"updatedRows\": 1, \"updatedColumns\": 1, \"updatedCells\": 1, \"modifiedTime\": \"2026-11-01T16:00:00Z\"}" + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01BaoS6rpnRHdejox1PMtF29", + "name": "mcp__apps__gcal_patch_event", + "input": { + "calendarId": "primary", + "eventId": "af7c4776398961d90c7b3274", + "patch": { + "description": "Slot held for Marcus's wedding favors: 40 Balsam & Cedar tins with custom labels, $16/tin + label cost, pending PackRight. If the labels fall through, offer Rae ~40 tins (Balsam/Cranberry) at $9 by Dec 8 instead (Tess's call, Nov 1). Also: Fireside 8 jars; Cranberry 10/15/2; Vanilla 4 tins/3 jars; gift-set tins 20 each F/B/C. ~63 lb. Cures Dec 5." + } + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "196264e2-2eeb-4368-8ce0-4326ba260d5d" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01BaoS6rpnRHdejox1PMtF29", + "type": "tool_result", + "content": { + "sha256": "1d857b982337c59aaa1fb3efb27d64bd71b06cb96d2cffb3dc1c5920f4aa8af3", + "chars": 1974 + } + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "72338b58-b4a8-4743-8975-8409c1e71215", + "timestamp": "2026-10-30T20:22:57.305Z", + "tool_use_result": { + "content": { + "sha256": "1d857b982337c59aaa1fb3efb27d64bd71b06cb96d2cffb3dc1c5920f4aa8af3", + "chars": 1974 + }, + "structuredContent": { + "result": { + "sha256": "3e368248ae37c0785a68c7032258a0264fd6b035f43ca1edde131b35b513fca6", + "chars": 1817 + } + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3azQBb9hRitvxJB4qS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01Nt6YcY4YEyBRQHrMtzie7R", + "name": "mcp__apps__gcal_create_event", + "input": { + "calendarId": "primary", + "event": { + "summary": "Chase PackRight: Marcus's custom labels + $136 order confirmation", + "description": "If the labels can be done by ~Dec 3: tell Marcus yes at $640 (40 x $16) + label cost, and order 1 more Balsam oil from Northwick. If not: tell Marcus we can't do the labels in time, and offer Rae ~40 tins at $9 by Dec 8 in the Nov 21 slot.", + "start": { + "date": "2026-11-03" + }, + "end": { + "date": "2026-11-04" + } + } + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 171, + "cache_read_input_tokens": 467184, + "cache_creation": { + "ephemeral_5m_input_tokens": 171, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "65bc96bc-2c75-46d7-90ce-c4f1f9d4bdb4" + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01Nt6YcY4YEyBRQHrMtzie7R", + "type": "tool_result", + "content": { + "sha256": "7227b1285ac45560f7766626055c43ad29202f84af2697156b08cdca6efbb9de", + "chars": 1707 + } + } + ] + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "c66c2419-1679-4424-9606-12d0f3f3112a", + "timestamp": "2026-10-30T20:22:59.180Z", + "tool_use_result": { + "content": { + "sha256": "7227b1285ac45560f7766626055c43ad29202f84af2697156b08cdca6efbb9de", + "chars": 1707 + }, + "structuredContent": { + "result": { + "sha256": "f3ad7622fab3ddc61404423deab62e2d78c71f6d7070b18c0d837725e595a5ec", + "chars": 1560 + } + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgZ3bpwCVF7vL5ucWzDgG", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": { + "sha256": "9369b704cf13ecd58235bec28eb77fe0de7f477b7f7ed2a2f7de44dcd2e2b39d", + "chars": 1568 + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 3245, + "cache_read_input_tokens": 467355, + "cache_creation": { + "ephemeral_5m_input_tokens": 3245, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 676 + }, + "input_transformations": [], + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "uuid": "0cb398c8-0d34-498a-a02c-6e778744039c" + }, + { + "type": "result", + "subtype": "success", + "is_error": false, + "api_error_status": null, + "duration_ms": 22747, + "duration_api_ms": 22494, + "ttft_ms": 2454, + "ttft_stream_ms": 2142, + "time_to_request_ms": 42, + "num_turns": 5, + "result": { + "sha256": "9369b704cf13ecd58235bec28eb77fe0de7f477b7f7ed2a2f7de44dcd2e2b39d", + "chars": 1568 + }, + "stop_reason": "end_turn", + "session_id": "daeab2f0-9b66-4fa3-ab68-c6ecae85beee", + "total_cost_usd": 0.7787085, + "usage": { + "input_tokens": 8, + "cache_creation_input_tokens": 5452, + "cache_read_input_tokens": 1399687, + "output_tokens": 1790, + "server_tool_use": { + "web_search_requests": 0, + "web_fetch_requests": 0 + }, + "service_tier": "standard", + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 5452 + }, + "inference_geo": "", + "iterations": [], + "speed": "standard" + }, + "modelUsage": { + "claude-opus-5-5": { + "inputTokens": 8, + "outputTokens": 1790, + "cacheReadInputTokens": 1399687, + "cacheCreationInputTokens": 5452, + "webSearchRequests": 0, + "costUSD": 0.7787085, + "contextWindow": 200000, + "maxOutputTokens": 32000 + } + }, + "permission_denials": [], + "terminal_reason": "completed", + "fast_mode_state": "off", + "uuid": "f51e8633-3eb7-4683-93e3-faa62ef7442d" + } + ] + ], + "run": { + "_turns": [ + { + "number": 1, + "message": { + "turn": 1, + "role": "user", + "text": "Book a table for four.", + "sent_at": null, + "author": null, + "written_by_triggers": [], + "user_sim": null, + "parts": [ + { + "kind": "text", + "text": "Book a table for four." + } + ], + "env_clock": null + }, + "reply": { + "turn": 1, + "role": "agent", + "text": "The table for four is booked.", + "sent_at": null, + "author": null, + "written_by_triggers": [], + "user_sim": null, + "parts": [ + { + "kind": "text", + "text": "The table for four is booked." + } + ], + "env_clock": null + }, + "_firings": [], + "_unavailable": {} + } + ], + "_firings": [], + "_logs": [ + "user_agent" + ], + "unavailable": {}, + "_entity_ids": {} + } + }, + "model_run_context_of_an_earlier_prompt": { + "calls": [ + { + "model": "judge-model", + "prompt": { + "sha256": "a1247524e2a26379fac8fb4001bb72e1fb5c6d01ce1a94538cbebf761dbe00f5", + "chars": 2389 + }, + "images": null, + "system": null, + "cached_prefix": null + } + ], + "files": {} + }, + "user_sim_chat": [ + { + "model": "fake/judge", + "names": [ + "delegation" + ], + "criteria": [ + { + "id": "c3", + "check": "Delegation (A4)", + "question": { + "sha256": "07a74a517c5ede32884839749db2046b29326d16aba8bb051e3fd707a8f8bd1c", + "chars": 1181 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "4ede798e87892d67a882570f4533659ebe65cbf0f4f53ba9416fa6c7b1a87824", + "chars": 14749 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + }, + { + "model": "fake/judge", + "names": [ + "ending" + ], + "criteria": [ + { + "id": "c7", + "check": "Ending (G4)", + "question": { + "sha256": "54238cbf0af0a835abdfe2d0cbf334aa3e873da78c2f9c31a2295b1e9fbb9989", + "chars": 569 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "c5d466e0430143e5ac7fa228a2edba41c0b39c680e179f259ae7d838e1c41ed3", + "chars": 14125 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + }, + { + "model": "fake/judge", + "names": [ + "guidance" + ], + "criteria": [ + { + "id": "c1", + "check": "Guidance (A1, A3)", + "question": { + "sha256": "9a9ea029a613397095f89de8b8f3aa230d8e22acf30a8c070f7259452fa6885d", + "chars": 2301 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "dbc581c3c05c9374ee336e78d9f415ac5b6e27a34a75ae3fdc8b686ca215cc5f", + "chars": 15879 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + }, + { + "model": "fake/judge", + "names": [ + "made_up" + ], + "criteria": [ + { + "id": "c6", + "check": "Made-up facts (A2)", + "question": { + "sha256": "0481faff49e323c5554f335302e31b91cbbd214ff6212d480450d22a94a55f31", + "chars": 1153 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "80f64deec2ee83543e76e66e9060e0bae88bbdf1f61bba779e78fb33037b862f", + "chars": 14725 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + }, + { + "model": "fake/judge", + "names": [ + "oversharing" + ], + "criteria": [ + { + "id": "c2", + "check": "Oversharing (G1, G2)", + "question": { + "sha256": "5d10ce962a3dd5c76c82a573361afae18e961e70a70f7e476a79336262a240f3", + "chars": 1675 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "4025ef816fcb04893eca08561fca72855c2928a6717acfc5fefee9009231f0c1", + "chars": 15245 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + }, + { + "model": "fake/judge", + "names": [ + "rogue" + ], + "criteria": [ + { + "id": "c5", + "check": "Rogue (G3, G5)", + "question": { + "sha256": "b1b197d9df1e6549cbaa0bcde06a8fccc9b7f32f8671cc8cc01f5b855ff89e5f", + "chars": 1216 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 1.0 + } + ], + "prompt": { + "sha256": "1af975f173b37595bde12f17b1a7b8122d50defecc87659065fc822fe957f180", + "chars": 14781 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + }, + { + "model": "fake/judge", + "names": [ + "style" + ], + "criteria": [ + { + "id": "c4", + "check": "Style (F1, F2, F3, F4)", + "question": { + "sha256": "fb856004c2a0f4f8c686370131a8672adfe0702967a7e4aceb27deb5b565bee9", + "chars": 1530 + }, + "graded_turns": [ + 1, + 2 + ], + "weight": 0.0 + } + ], + "prompt": { + "sha256": "30c4ca22a5dd9e2f7660f2b5354204f668189dcc54a0c5e143a59da6dc2b48ad", + "chars": 15095 + }, + "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", + "cached_prefix": { + "sha256": "268023e8ef3618b94aff6b472a8fd2dc522e07d69b7d19223b4a3c0b316f763d", + "chars": 13255 + } + } + ] +} diff --git a/tst/data/verifier_golden/judge_requests.json b/tst/data/verifier_golden/judge_requests.json new file mode 100644 index 000000000..fe9aea6da --- /dev/null +++ b/tst/data/verifier_golden/judge_requests.json @@ -0,0 +1,914 @@ +{ + "gpt_through_the_endpoint": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "claude_through_the_endpoint": { + "requests": [ + { + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "anthropic_prefixed_through_a_base_without_v1": { + "requests": [ + { + "model": "anthropic/claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "gemini_through_the_endpoint_as_openai": { + "requests": [ + { + "model": "openai/gemini-2.5-flash", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "a_name_litellm_cant_place": { + "requests": [ + { + "model": "openai/qwen3:8b", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "a_prefixed_name_with_no_endpoint": { + "requests": [ + { + "model": "openai/gpt-4o", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "params_and_headers": { + "requests": [ + { + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "custom_llm_provider": "openai", + "temperature": 0, + "extra_headers": { + "X-Team": "evals" + }, + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "the_judges_own_url_and_key": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "judge-secret", + "api_base": "https://judge.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "the_runs_url_on_the_endpoint": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "extra_headers": { + "X-Team": "evals" + }, + "api_base": "https://GW.test/v1/", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "the_steps_url_elsewhere_with_the_runs_key": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "run-key", + "api_base": "https://step.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "a_url_elsewhere_without_a_key": { + "requests": [], + "raised": "rubrics_verifier 'v': x.test isn't the configured model endpoint, so the configured key isn't sent there, and no key was given with it. Pass one with it: judge_litellm_api_key, or litellm_api_key, in the run's overrides. Any non-empty value works for a server that needs no key." + }, + "screenshots": { + "requests": [ + { + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "THE RUN then the check" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=" + } + } + ] + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "screenshots_refused_with_the_schema": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "THE RUN then the check" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=" + } + } + ] + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + }, + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "THE RUN then the check" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=" + } + } + ] + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test/v1" + } + ], + "raised": null + }, + "a_system_message_and_a_cached_run": { + "requests": [ + { + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "system", + "content": "You check the run." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "THE RUN", + "cache_control": { + "type": "ephemeral" + } + }, + { + "type": "text", + "text": " then the check" + } + ] + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "a_cached_run_on_a_model_without_a_prompt_cache": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 1000, + "api_key": "configured-key", + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + }, + "a_timeout": { + "requests": [ + { + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "THE RUN then the check" + } + ], + "timeout": 42, + "api_key": "configured-key", + "api_base": "https://gw.test/v1", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "rubrics_verification", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number" + }, + "justification": { + "type": "string" + } + }, + "required": [ + "id", + "score", + "justification" + ], + "additionalProperties": false + } + } + }, + "required": [ + "results" + ], + "additionalProperties": false + }, + "strict": true + } + } + } + ], + "raised": null + } +} diff --git a/tst/integration/cli/run_bundle_built_envs_test.py b/tst/integration/cli/run_bundle_built_envs_test.py index a2d862105..9e2e17d52 100644 --- a/tst/integration/cli/run_bundle_built_envs_test.py +++ b/tst/integration/cli/run_bundle_built_envs_test.py @@ -397,9 +397,10 @@ def test_environments_and_a_universe_written_from_a_bundle_load_into_an_env_thro # Checks each items server CONFIG names holds exactly the items loaded into it, read from its export-state route behind -# the multi's gateway. CONFIG goes above it. +# the multi's gateway, with the control token the run's grant gives. CONFIG goes above it. VERIFY_SEEDED = ''' import httpx +from agentenv_protocol.client import request_headers async def verify(mcp_url): @@ -407,7 +408,8 @@ async def verify(mcp_url): results = [] async with httpx.AsyncClient(timeout=30) as client: for server, items in CONFIG["servers"].items(): - response = await client.get(f"{gateway}/svc/mcp-{server}/export-state") + url = f"{gateway}/svc/mcp-{server}/export-state" + response = await client.get(url, headers=request_headers(url)) listed = response.json() if response.status_code == 200 else {} description = f"{server} holds what it was seeded with, {items}: {response.status_code} {listed}" results.append({"id": server, "result": listed.get("items") == items, "description": description}) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 22b58e4ed..c118d5062 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -20,7 +20,7 @@ from click.testing import CliRunner import agent_env -from agent_env.artifact import FileArtifactUniverse +from agent_env.artifact import Artifact, FileArtifactUniverse from agent_env.artifact.store import reset_artifact_store from agent_env.bundle.hub import HubSource from agent_env.bundle.installed import find_bundle @@ -35,6 +35,7 @@ ) from agent_env.providers.sandbox_providers.sandbox_provider import reset_sandbox_provider from agent_env.store import Filter, LocalSqliteDocumentStore +from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing from agent_env.task import Task from tst.integration.utils.registry import REGISTRY_IMAGE @@ -179,6 +180,27 @@ def test_a_task_file_naming_its_headline_verifiers_runs_and_its_stored_task_keep assert Task.get("@local/~/my-hello/hello", 1).headline_verifiers == ["hello"] +def test_a_task_file_with_a_field_its_step_doesnt_read_is_refused_before_anything_is_written(bundle, quiet_logs): + steps = json.loads((bundle / "tasks/hello.json").read_text()) + (bundle / "tasks/hello.json").write_text(json.dumps([*steps[:2], {**steps[2], "min_passing_scor": 1.0}])) + + refused = CliRunner().invoke(cli, ["run", str(bundle)]) + + assert refused.exit_code != 0 + assert ("tasks/hello.json: step 'hello': verify_sandbox doesn't read 'min_passing_scor' (did you mean " + "'min_passing_score'?), so it would be ignored") in refused.output + with namespace_routing(): + for get in (lambda: Task.get("@local/~/my-hello/hello"), lambda: Artifact.get("@local/~/my-hello/greeting")): + with pytest.raises(NotFoundError): + get() + + (bundle / "tasks/hello.json").write_text(json.dumps([*steps, {**steps[2], "id": "again", "verifier_id": "again"}])) + output = _run(bundle) + + assert "tasks/hello.json: warning: 2 verifiers and no headline_verifiers" in output + assert "tasks/hello.json v1: passed" in output + + @skip_without_gnu_stat() def test_a_bundle_task_collects_from_its_sandbox_into_a_universe_named_after_the_run(bundle, quiet_logs): steps = json.loads((bundle / "tasks/hello.json").read_text()) diff --git a/tst/integration/env/envs/multi_env_local_test.py b/tst/integration/env/envs/multi_env_local_test.py index 50c369c60..4b23a0f69 100644 --- a/tst/integration/env/envs/multi_env_local_test.py +++ b/tst/integration/env/envs/multi_env_local_test.py @@ -55,13 +55,13 @@ async def test_a_multi_env_deploys_restores_loads_and_tears_down_on_local_compos base_url = await legacy_protocol.v1_base_url(deployed, deployed.gateway_url, "items") assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]} slack_state = f"{legacy_protocol.environment_base_url(deployed.gateway_url, 'slack')}/export-state" - assert len(httpx.get(slack_state, timeout=30).json()["channels"]) == 10 + assert len(httpx.get(slack_state, headers=deployed.control_headers, timeout=30).json()["channels"]) == 10 # A multi env without computers resets its servers, each as it resets alone, and fails if one doesn't. await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute( TaskStepContext(deployed_envs=[deployed])) assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": []} - assert httpx.get(slack_state, timeout=30).json()["channels"] == [] + assert httpx.get(slack_state, headers=deployed.control_headers, timeout=30).json()["channels"] == [] await restored.close() assert _stack(work_dir) == "" diff --git a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py index ed3125720..3216fd1e6 100644 --- a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py +++ b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py @@ -35,6 +35,7 @@ from agent_env.store.image_store import LocalRegistryImageStore from agent_env.store.object_store.local.grant_server import grant_server from agent_env.task import Task +from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep @@ -100,6 +101,11 @@ def local_registry(monkeypatch, tmp_path): reset_config() +def _prescribing_atif() -> TaskStepContext: + """A run that prescribes atif/1, so its prompt steps have the agents they prompt build it.""" + return TaskStepContext(metadata={"user_overrides": {"trajectory_formats": [ATIF]}}) + + def _deploy(agent, name: str, **snapshot) -> DeployAgentTaskStep: return DeployAgentTaskStep( id=f"deploy-{name}", version=None, env_ids=[], a2a_agent_id=agent.id, a2a_agent_version=agent.version, @@ -149,6 +155,8 @@ async def test_each_turn_is_stored_live_under_its_own_prefix_and_joins_to_its_fi assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") finals = context.prompt_responses[-1].target_agent_per_turn_trajectory_object_urls assert len(finals) == 2 and all(finals) + assert context.prompt_responses[-1].target_agent_per_turn_trajectories == [ + {"native_format": "agentenv-echo-agent/v1", "atif": None}] * 2 turns = [store.get_object_key(url).removeprefix(f"{prefix_key}trajectory-").removesuffix(".json") for url in finals] assert len(set(turns)) == 2 for turn, final_url in zip(turns, finals): @@ -177,7 +185,7 @@ async def test_a_deployed_agent_serves_its_native_format_and_atif_and_refuses_an ), ]) - context = await task.run() + context = await task.run(context=_prescribing_atif()) assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") [deployed] = context.deployed_agents @@ -219,7 +227,7 @@ async def test_a_snapshot_carries_the_turn_count_to_the_fresh_agent_it_is_restor _prompt("three", "restored", restored), ]) - context = await task.run() + context = await task.run(context=_prescribing_atif()) assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") store = get_config().get_object_store() @@ -246,7 +254,7 @@ async def test_a_model_key_a_command_prints_is_redacted_in_atif_and_kept_in_the_ prompt="run printenv LITELLM_API_KEY", context_id=context_id), ]) - context = await task.run() + context = await task.run(context=_prescribing_atif()) assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") [deployed] = context.deployed_agents @@ -285,14 +293,23 @@ async def test_a_deployed_agent_pushes_its_copys_media_under_exact_object_grants parts = [{"kind": "text", "text": "look"}, {"kind": "file", "file": {"bytes": base64.b64encode(image).decode(), "mimeType": "image/png", "name": "shot.png"}}] - task_id, _ = await send_a2a_message(deployed.a2a_url, parts, f"m-{suffix}", context_id, 60) - await poll_a2a_task(deployed.a2a_url, task_id, 120, 1) trajectory = A2AAgent.find_extension(deployed.a2a_card, A2AAgent.EXT_TRAJECTORY) _, media_endpoint = A2AAgent.operation(trajectory, "media") + _, prescribe_endpoint = A2AAgent.operation(trajectory, "prescribe") + read_url = deployed.a2a_url + trajectory["params"]["endpoint"] + unprescribed, _ = await send_a2a_message(deployed.a2a_url, parts, f"m0-{suffix}", f"unprescribed-{suffix}", 60) + await poll_a2a_task(deployed.a2a_url, unprescribed, 120, 1) + async with httpx.AsyncClient(timeout=60, headers=deployed.control_headers) as client: + refused = await client.post(read_url, json={"task_id": unprescribed, "format": ATIF}) + prescribed = (await client.post(deployed.a2a_url + prescribe_endpoint, json={"formats": [ATIF]})).json() + with pytest.raises(httpx.HTTPStatusError, match="HTTP 409: format_not_prescribed"): + raise_for_extension_status(refused, operation="trajectory read") + assert prescribed == {"formats": ["agentenv-echo-agent/v1", ATIF]} + task_id, _ = await send_a2a_message(deployed.a2a_url, parts, f"m-{suffix}", context_id, 60) + await poll_a2a_task(deployed.a2a_url, task_id, 120, 1) store = get_config().get_object_store() async with httpx.AsyncClient(timeout=60, headers=deployed.control_headers) as client: - atif = (await client.post(deployed.a2a_url + trajectory["params"]["endpoint"], - json={"task_id": task_id, "format": ATIF})).json() + atif = (await client.post(read_url, json={"task_id": task_id, "format": ATIF})).json() [path] = media_paths(atif["trajectory"]) url = store.object_url(f"atif-media-{suffix}/{path}") grant = write_object(store, url, media_type="image/png", max_bytes=len(image)) diff --git a/tst/unit/a2a_agent/object_transfer_test.py b/tst/unit/a2a_agent/object_transfer_test.py index 39f2bdcff..3f7d7f173 100644 --- a/tst/unit/a2a_agent/object_transfer_test.py +++ b/tst/unit/a2a_agent/object_transfer_test.py @@ -10,6 +10,7 @@ from agent_env.a2a_agent import object_transfer from agent_env.a2a_agent.object_transfer import ( + TrajectoryFormats, FetchedTrajectory, TrajectoryUpload, ObjectLimits, @@ -222,6 +223,48 @@ def answer(request: httpx.Request) -> httpx.Response: ] +def test_a_cards_trajectory_formats_are_read_from_its_params_and_an_older_card_has_none(): + offered = {"params": {"formats": ["claude-code-session/1", "atif/1"], "native": "claude-code-session/1", + "default": "claude-code-session/1"}} + assert TrajectoryFormats.of(offered) == TrajectoryFormats( + ("claude-code-session/1", "atif/1"), "claude-code-session/1", "claude-code-session/1") + for older in (None, {}, {"params": {"endpoint": "/ext/trajectory"}}, {"params": {"formats": []}}): + assert TrajectoryFormats.of(older) == TrajectoryFormats((), None, None) + + +@pytest.mark.asyncio +async def test_a_trajectory_get_names_a_format_only_when_given_and_reports_the_one_served(tmp_path, monkeypatch): + sent = [] + real_client = httpx.AsyncClient + + def answer(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + sent.append(body) + served = {"format": body["format"]} if "format" in body else {} + if "objects" in body: + return httpx.Response(200, json={"objects": {"trajectory": {"size_bytes": 2}}, **served}) + return httpx.Response(200, json={"trajectory": [], **served}) + + monkeypatch.setattr( + object_transfer.httpx, + "AsyncClient", + lambda **kwargs: real_client(transport=httpx.MockTransport(answer), **kwargs), + ) + store = GrantingObjectStore(str(tmp_path)) + url = store.object_url("t/trajectory.json") + endpoint = "https://agent.test/ext/trajectory" + + unnamed = await fetch_trajectory(endpoint, {"task_id": "t"}) + inline = await fetch_trajectory(endpoint, {"task_id": "t"}, format="echo/1") + uploaded = await fetch_trajectory(endpoint, {"task_id": "t"}, upload=TrajectoryUpload.to(store, url), + format="echo/1") + + assert unnamed == FetchedTrajectory(inline=[]) + assert inline == FetchedTrajectory(inline=[], format="echo/1") + assert uploaded == FetchedTrajectory(object_url=url, format="echo/1") + assert [set(body) for body in sent] == [{"task_id"}, {"task_id", "format"}, {"task_id", "format", "objects"}] + + @pytest.mark.asyncio async def test_a_trajectory_upload_grants_only_the_trajectory_so_a_pinned_agent_accepts_it(tmp_path, monkeypatch): sent = [] diff --git a/tst/unit/a2a_agent/test_trajectory_follower.py b/tst/unit/a2a_agent/test_trajectory_follower.py index dfebf921e..99108b1b7 100644 --- a/tst/unit/a2a_agent/test_trajectory_follower.py +++ b/tst/unit/a2a_agent/test_trajectory_follower.py @@ -42,17 +42,21 @@ def __init__(self, *answers: httpx.Response) -> None: self.answers = list(answers) self.afters: list[int] = [] self.tokens: list[str | None] = [] + self.formats: list[str | None] = [] def client(self) -> httpx.AsyncClient: return httpx.AsyncClient(transport=httpx.MockTransport(self._answer)) def _answer(self, request: httpx.Request) -> httpx.Response: - self.afters.append(json.loads(request.content)["after"]) + body = json.loads(request.content) + self.afters.append(body["after"]) + self.formats.append(body.get("format")) self.tokens.append(request.headers.get("agentenv-control-token")) return self.answers.pop(0) -async def _follow(agent: _Agent, headers: dict[str, str] | None = None) -> list[TrajectoryBatch]: +async def _follow(agent: _Agent, headers: dict[str, str] | None = None, format: str | None = None + ) -> list[TrajectoryBatch]: batches: list[TrajectoryBatch] = [] async def sink(batch: TrajectoryBatch) -> None: @@ -60,7 +64,8 @@ async def sink(batch: TrajectoryBatch) -> None: async with agent.client() as client: await follow_trajectory( - _ENDPOINT, "task-1", sink, poll_interval_seconds=0, stop=asyncio.Event(), client=client, headers=headers + _ENDPOINT, "task-1", sink, poll_interval_seconds=0, stop=asyncio.Event(), client=client, headers=headers, + format=format, ) return batches @@ -116,6 +121,18 @@ async def test_every_read_carries_the_headers_it_was_given(): assert agent.tokens == ["tok", "tok"] +@pytest.mark.asyncio +async def test_every_read_names_the_format_it_was_given_and_none_without_one(): + pages = [_page(0, [{"i": 0}], "running"), _page(1, [], "completed")] + named, unnamed = _Agent(*(httpx.Response(200, json=page) for page in pages)), _Agent( + *(httpx.Response(200, json=page) for page in pages)) + + await _follow(named, format="echo/1") + await _follow(unnamed) + + assert named.formats == ["echo/1", "echo/1"] and unnamed.formats == [None, None] + + @pytest.mark.asyncio async def test_an_ended_task_with_nothing_new_still_gets_a_last_batch(): agent = _Agent(httpx.Response(200, json=_page(0, [], "failed"))) diff --git a/tst/unit/agents/codex/test_config_and_mcp.py b/tst/unit/agents/codex/test_config_and_mcp.py index ded20a4cc..4b730b46a 100644 --- a/tst/unit/agents/codex/test_config_and_mcp.py +++ b/tst/unit/agents/codex/test_config_and_mcp.py @@ -87,6 +87,7 @@ async def exercise(): config = tomllib.loads((Path(os.environ["CODEX_HOME"]) / "config.toml").read_text()) assert config == { "features": {"multi_agent_v2": True}, + "tools": {"update_plan": {"enabled": True}}, "model_provider": "litellm", "model_providers": {"litellm": { "name": "LiteLLM proxy", @@ -98,11 +99,12 @@ async def exercise(): "stream_idle_timeout_ms": 60000, "env_http_headers": {"x-litellm-customer-id": "LITELLM_CUSTOMER_ID", "x-litellm-tags": "LITELLM_TAGS"}, "http_headers": {"X-Gateway": "g"}, + "capabilities": {"remote_compaction": "v2"}, }}, } -def test_without_an_endpoint_only_the_features_are_written(codex, monkeypatch): +def test_without_an_endpoint_no_model_provider_is_written(codex, monkeypatch): monkeypatch.delenv("LITELLM_BASE_URL", raising=False) async def exercise(): @@ -112,7 +114,7 @@ async def exercise(): asyncio.run(exercise()) config = tomllib.loads((Path(os.environ["CODEX_HOME"]) / "config.toml").read_text()) - assert config == {"features": {"multi_agent_v2": True}} + assert config == {"features": {"multi_agent_v2": True}, "tools": {"update_plan": {"enabled": True}}} def test_turns_of_different_roles_share_one_config_and_each_cli_gets_its_own_role(codex, monkeypatch): diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index f20354e69..ed5d3b8ba 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -281,11 +281,11 @@ def test_the_cli_says_so_when_no_bundles_are_installed_and_needs_a_bundle_for_it assert CliRunner().invoke(cli, ["run"]).output == "No bundles are installed. agent-env run PATH runs a folder.\n" for option in (["--task", "t"], ["--keep"], ["--dry-run"], ["--user-model", "m"], ["--judge-model", "m"], - ["--agent", "pi"], ["--set", "envs/desktop.driver=reference"], ["--no-emulate"]): + ["--agent", "pi"], ["--set", "envs/desktop.driver=reference"], ["--no-emulate"], ["--skip-lint"]): usage = CliRunner().invoke(cli, ["run", *option]) assert usage.exit_code == 2 - assert ("--task, --eval, --model, --user-model, --judge-model, --agent, --set, --sandbox, --keep, --dry-run " - "and --no-emulate need a BUNDLE to run") in " ".join(usage.output.split()) + assert ("--task, --eval, --model, --user-model, --judge-model, --agent, --set, --sandbox, --keep, --dry-run, " + "--no-emulate and --skip-lint need a BUNDLE to run") in " ".join(usage.output.split()) def test_the_cli_runs_an_installed_bundle_by_name_with_ids_rooted_at_its_package(site, monkeypatch, quiet_logs, diff --git a/tst/unit/bundle/parse_test.py b/tst/unit/bundle/parse_test.py index 5fe85ab29..bd05dfeff 100644 --- a/tst/unit/bundle/parse_test.py +++ b/tst/unit/bundle/parse_test.py @@ -657,6 +657,13 @@ def test_a_task_files_object_takes_only_steps_and_a_list_of_headline_verifiers(s f"tasks/b.json: {TASK_SHAPE}", ) + def test_a_step_whose_id_or_type_is_an_array_or_object_is_refused(self, tmp_path): + root = build(tmp_path / "b", {"tasks/a.json": '[{"id": "box", "type": "x"}, {"id": ["box"], "type": "x"}]', + "tasks/b.json": '[{"id": "box", "type": {"name": "x"}}]', + "tasks/c.json": '[{"id": 1, "type": "x"}]'}) + assert problems(root) == ("tasks/a.json: step #1's id is a JSON array; it must be a string", + "tasks/b.json: step #0's type is a JSON object; it must be a string") + @pytest.mark.parametrize("rel, content", [ ("tasks/t.json", "[" * 100_000 + "]" * 100_000), ("evals/e.toml", "a = " + "[" * 5000 + "]" * 5000 + "\n"), diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 558a17efc..4604b55b0 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -50,6 +50,9 @@ ARM64_HERE = (*LOCAL, "linux/arm64") # built here on Apple silicon, as the infra bootstrap builds it HERE = "localhost:5000/local/img:v1 is in a registry on this machine, which a sandbox elsewhere can't pull from" +# The lint warning a task with no verifier gets, which a run says before it writes. +_NO_GRADE = "tasks/t.json: warning: no step grades this task, so a run has no score" + @pytest.fixture def bundle_dir(tmp_path, monkeypatch, local_stores): @@ -1044,7 +1047,7 @@ async def deploy(self, context): run_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) - assert events[:3] == [("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"]), + assert events[:4] == [("say", _NO_GRADE), ("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"]), ("say", "[tasks/t.json] step 1/1 env (deploy_env)")] assert ("deploy", "crm") in events @@ -1065,7 +1068,7 @@ async def deploy(self, context): materialization = _write_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) - assert events == [("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"])] + assert events == [("say", _NO_GRADE), ("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"])] assert [done.write.kind for done in materialization.writes] == [BundleKind.TASK] # The walk resolves each deploy's provider as its step does diff --git a/tst/unit/bundle/register_test.py b/tst/unit/bundle/register_test.py index 2be9159d9..917211e18 100644 --- a/tst/unit/bundle/register_test.py +++ b/tst/unit/bundle/register_test.py @@ -395,6 +395,22 @@ def test_the_cli_says_what_it_would_write_asks_before_a_shared_store_and_dry_run assert [task.version for task in [Task.get("triage-t")]] == [1] +def test_the_cli_says_each_lint_warning_once_and_a_lint_error_refuses_before_anything_is_written(home): + root = layout(home / "triage", TRIAGE) + warning = " tasks/t.json: warning: no step grades this task, so a run has no score" + + dry = CliRunner().invoke(cli, ["bundle", "register", str(root), "--dry-run"]) + written = CliRunner().invoke(cli, ["bundle", "register", str(root), "--yes"]) + (root / "tasks/t.json").write_text(json.dumps([BOX, {**_load("greet", "greeting"), "destinaton_path": "/app"}])) + refused = CliRunner().invoke(cli, ["bundle", "register", str(root), "--yes"]) + + assert dry.output.splitlines().count(warning) == written.output.splitlines().count(warning) == 1 + assert refused.exit_code == 1 + assert ("tasks/t.json: step 'greet': load_artifact doesn't read 'destinaton_path' (did you mean " + "'destination_path'?), so it would be ignored") in refused.output + assert Task.get("triage-t").version == 1 + + def test_a_bundle_with_no_tasks_registers_every_entity(home): registered = register_bundle(layout(home / "library", {"artifacts/greeting/hello.txt": "hello\n"})) diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py index de2abdc4d..fec9aa31e 100644 --- a/tst/unit/bundle/run_test.py +++ b/tst/unit/bundle/run_test.py @@ -553,6 +553,7 @@ def test_the_cli_dry_run_prints_what_the_run_would_write_and_run_and_exits_0(bun assert result.output == ( DRY_RUN + f"{CONFIGURED}\n" + "tasks/code.json: warning: no step grades this task, so a run has no score\n" "artifacts/script: v1 (new)\n" "tasks/a.json: v1 (new)\n" "tasks/b.json: v1 (new)\n" @@ -596,6 +597,7 @@ def test_the_cli_dry_run_reads_another_bundles_entity_and_says_why_an_agent_pinn assert result.output.startswith( DRY_RUN + f"{CONFIGURED}\n" + "tasks/agent.json: warning: no step grades this task, so a run has no score\n" f"agents/solver: v2 (artifact {image} has a new version in the store (v1 → v2))\n" "tasks/agent.json: v1, unchanged\n" "\n" diff --git a/tst/unit/cli/eval_run_teardown_test.py b/tst/unit/cli/eval_run_teardown_test.py index e1025a91b..ceea94fc8 100644 --- a/tst/unit/cli/eval_run_teardown_test.py +++ b/tst/unit/cli/eval_run_teardown_test.py @@ -22,7 +22,7 @@ class _Deploys: """Records a sandbox on the fake backend, then naps.""" def __init__(self, id, nap): - self.id, self.version, self.steps, self.nap = id, 1, [], nap + self.id, self.version, self.steps, self.nap, self.headline_verifiers = id, 1, [], nap, None async def run(self, context, **_): context.deployed_sandboxes.append(DeployedSandbox( diff --git a/tst/unit/cli/run_role_models_test.py b/tst/unit/cli/run_role_models_test.py index b4837aefe..65de2a589 100644 --- a/tst/unit/cli/run_role_models_test.py +++ b/tst/unit/cli/run_role_models_test.py @@ -21,7 +21,7 @@ class _Recorded: """Records each context it runs with.""" - id, version, steps, requires = "t", 1, [], None + id, version, steps, requires, headline_verifiers = "t", 1, [], None, None def __init__(self): self.contexts = [] diff --git a/tst/unit/cli/submit_runs_test.py b/tst/unit/cli/submit_runs_test.py index 42805f6cf..dd2bfa8b2 100644 --- a/tst/unit/cli/submit_runs_test.py +++ b/tst/unit/cli/submit_runs_test.py @@ -28,7 +28,7 @@ class _Task: - id, version, steps, requires = "t", 1, [], None + id, version, steps, requires, headline_verifiers = "t", 1, [], None, None def __init__(self, nap=0.0, failed_steps=None): self.nap, self.failed_steps, self.contexts, self.calls = nap, failed_steps, [], [] diff --git a/tst/unit/cli/test_task_create_preflight.py b/tst/unit/cli/test_task_create_preflight.py index 19d336790..dc41c611e 100644 --- a/tst/unit/cli/test_task_create_preflight.py +++ b/tst/unit/cli/test_task_create_preflight.py @@ -97,3 +97,28 @@ def test_skip_validation_saves_and_still_prints_the_problems(tmp_path, store): assert "Saving anyway" in result.output put_step.assert_called_once() put_task.assert_called_once() + + +def test_a_field_no_step_reads_is_refused_naming_the_one_it_meant_and_persists_nothing(tmp_path, store): + put_step, put_task = store + path = tmp_path / "steps.json" + path.write_text(json.dumps([{"id": "c1", "type": "run_code", "script_artifact_id": "a", "agent_nme": "x"}])) + + result = CliRunner().invoke(create, [str(path), "--id", "t", "--skip-validation"]) + + assert result.exit_code == 1 + assert "Error: step 'c1': run_code doesn't read 'agent_nme' (did you mean 'agent_name'?)" in result.output + put_step.assert_not_called() + put_task.assert_not_called() + + +def test_a_lint_error_is_a_preflight_problem_and_a_lint_warning_is_only_printed(tmp_path, store): + put_step, put_task = store + with patch("agent_env.artifact.Artifact.get", side_effect=NotFoundError("nope")): + refused = CliRunner().invoke(create, [str(_object_file(tmp_path, headline_verifiers=["c1"])), "--id", "t"]) + + assert refused.exit_code == 1 + assert "Warning: no step grades this task, so a run has no score" in refused.output + assert " - headline_verifiers names 'c1', a run_code step, which grades nothing" in refused.output + put_step.assert_not_called() + put_task.assert_not_called() diff --git a/tst/unit/config/test_describe.py b/tst/unit/config/test_describe.py index ea7a17c0a..ed5cf728b 100644 --- a/tst/unit/config/test_describe.py +++ b/tst/unit/config/test_describe.py @@ -286,6 +286,9 @@ def test_render_puts_the_file_and_every_section_on_screen(tmp_path): [agents] default_a2a_agent_id = "an-agent" +[trajectories] +formats = ["atif/1"] + [sandbox] default = "local" @@ -319,7 +322,7 @@ def test_render_puts_the_file_and_every_section_on_screen(tmp_path): _ALIASED = ("document", "object", "image", "secret", "runner") _EVERY_TABLE = ("document", "object", "image", "secret", "runner", "model", "conversations", - "agents", "sandbox", "state", "envs", "artifacts", "task_steps", "gateway", "explorer") + "agents", "trajectories", "sandbox", "state", "envs", "artifacts", "task_steps", "gateway", "explorer") @pytest.mark.parametrize("toml", [None, MONGO_TOML, FULL_SURFACE_TOML]) diff --git a/tst/unit/config/test_model_call.py b/tst/unit/config/test_model_call.py new file mode 100644 index 000000000..4168233c9 --- /dev/null +++ b/tst/unit/config/test_model_call.py @@ -0,0 +1,229 @@ +"""Core's one model call: its route and key, and its record of what it called and what it cost. A call's cost is what +the endpoint billed, a LiteLLM proxy's ``x-litellm-response-cost`` header, else LiteLLM's price map's figure, never +LiteLLM's own estimate. The real LiteLLM calls a loopback server on the OpenAI and the Anthropic routes.""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace + +import litellm +import pytest +from pytest_socket import enable_socket + +from agent_env.config import model_call +from agent_env.config.errors import ConfigError +from agent_env.config.model_pricing import litellm_prices +from tst.util.config import install_model_config + +_HOW = "judge_litellm_api_key in the run's overrides" +_USER = [{"role": "user", "content": "grade it"}] + + +class _Endpoint(BaseHTTPRequestHandler): + """Answers OpenAI's chat completions and Anthropic's messages with ``usage``, and the ``billed`` header when set.""" + + billed: str | None = None + usage: dict = {} + + def log_message(self, *args) -> None: + pass + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers.get("content-length") or 0)) or b"{}") + if self.path.endswith("/messages"): + answer = {"id": "m", "type": "message", "role": "assistant", "model": body.get("model"), + "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", "usage": self.usage} + else: + answer = {"id": "c", "object": "chat.completion", "created": 0, "model": body.get("model"), + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop"}], "usage": self.usage} + data = json.dumps(answer).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + if self.billed is not None: + self.send_header("x-litellm-response-cost", self.billed) + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +@pytest.fixture(autouse=True) +def _disable_network(): + """Override the suite-wide socket block (tst/unit/conftest.py) for this module: the endpoint is on loopback.""" + enable_socket() + yield + + +@pytest.fixture +def endpoint(): + server = ThreadingHTTPServer(("127.0.0.1", 0), _Endpoint) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + _Endpoint.billed, _Endpoint.usage = None, {} + + +def _configure(monkeypatch, base_url: str | None) -> None: + install_model_config(monkeypatch, secrets={"own": "own-key"}, base_url=base_url, api_key="configured-key") + + +async def _call(monkeypatch, endpoint: str, model: str, *, billed: str | None, usage: dict) -> model_call.ModelReply: + _configure(monkeypatch, endpoint + "/v1") + _Endpoint.billed, _Endpoint.usage = billed, usage + route = model_call.resolve_route(model, api_key=None, api_base=None, who="judge", key_how=_HOW) + return await model_call.complete(route, _USER, timeout=30) + + +_OPENAI_USAGE = {"prompt_tokens": 1000, "completion_tokens": 100, "total_tokens": 1100} +_ANTHROPIC_USAGE = {"input_tokens": 100, "cache_read_input_tokens": 900, "output_tokens": 50} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model, usage", [("gpt-4.1-mini", _OPENAI_USAGE), ("claude-sonnet-4-6", _ANTHROPIC_USAGE)], + ids=["openai route", "anthropic route"]) +async def test_a_call_a_proxy_bills_costs_what_it_billed(monkeypatch, endpoint, model, usage): + reply = await _call(monkeypatch, endpoint, model, billed="0.0123", usage=usage) + + assert reply.content == "ok" and reply.model == model and reply.endpoint_host == endpoint.removeprefix("http://") + assert reply.fields() | {"usage": None} == { + "usage": None, "model": model, "endpoint_host": endpoint.removeprefix("http://"), "cost_usd": 0.0123, + "cost_status": "reported", "cost_source": "provider", "reported_cost_usd": 0.0123} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model, usage, usd", [ + ("gpt-4.1-mini", _OPENAI_USAGE, (1000 * 0.4 + 100 * 1.6) / 1e6), + ("claude-sonnet-4-6", _ANTHROPIC_USAGE, (100 * 3 + 900 * 0.3 + 50 * 15) / 1e6), +], ids=["openai route", "anthropic route, its cache reads at their own rate"]) +async def test_a_call_no_one_billed_is_priced_from_litellms_map(monkeypatch, endpoint, model, usage, usd): + reply = await _call(monkeypatch, endpoint, model, billed=None, usage=usage) + + assert reply.cost.status == "priced" and reply.cost.source == litellm_prices().source + assert reply.cost.usd == pytest.approx(usd, rel=1e-9) and reply.cost.reported_usd is None + assert reply.usage["completion_tokens"] == usage.get("completion_tokens", usage.get("output_tokens")) + + +@pytest.mark.asyncio +async def test_a_model_the_map_lacks_costs_what_it_was_billed_and_otherwise_nothing(monkeypatch, endpoint): + billed = await _call(monkeypatch, endpoint, "qwen3:8b", billed="0.002", usage=_OPENAI_USAGE) + unbilled = await _call(monkeypatch, endpoint, "qwen3:8b", billed=None, usage=_OPENAI_USAGE) + + assert (billed.cost.usd, billed.cost.status, billed.cost.source) == (0.002, "reported", "provider") + assert unbilled.cost.fields() == {"cost_usd": None, "cost_status": "unpriced", "cost_source": None, + "reported_cost_usd": None, "unpriced_reason": "no_price"} + + +def _answer(monkeypatch, hidden: dict) -> None: + """Every call answered with LiteLLM's own response, its ``_hidden_params`` replaced by ``hidden``.""" + async def acompletion(**kwargs): + response = litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "ok"}}], + usage=litellm.Usage(prompt_tokens=1000, completion_tokens=100)) + response._hidden_params = hidden + return response + + monkeypatch.setattr(litellm, "acompletion", acompletion) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model, status, usd", [("gpt-4.1-mini", "priced", 0.00056), ("qwen3:8b", "unpriced", None)]) +async def test_litellms_own_estimate_is_never_the_cost(monkeypatch, model, status, usd): + _configure(monkeypatch, "https://gw.test/v1") + _answer(monkeypatch, {"response_cost": 9.99, "additional_headers": {}}) + route = model_call.resolve_route(model, api_key=None, api_base=None, who="judge", key_how=_HOW) + + reply = await model_call.complete(route, _USER, timeout=30) + + assert reply.cost.status == status and reply.cost.usd == pytest.approx(usd) and reply.cost.reported_usd is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("header", ["n/a", "-1", "nan", None]) +async def test_a_header_that_isnt_a_cost_is_no_bill(monkeypatch, header): + _configure(monkeypatch, "https://gw.test/v1") + _answer(monkeypatch, {"additional_headers": {"llm_provider-x-litellm-response-cost": header}}) + route = model_call.resolve_route("gpt-4.1-mini", api_key=None, api_base=None, who="judge", key_how=_HOW) + + reply = await model_call.complete(route, _USER, timeout=30) + + assert reply.cost.status == "priced" and reply.cost.reported_usd is None + + +@pytest.mark.asyncio +async def test_a_response_without_usage_is_unpriced_for_want_of_tokens(monkeypatch): + _configure(monkeypatch, "https://gw.test/v1") + + async def acompletion(**kwargs): + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=None))]) + + monkeypatch.setattr(litellm, "acompletion", acompletion) + route = model_call.resolve_route("gpt-4.1-mini", api_key=None, api_base=None, who="judge", key_how=_HOW) + + reply = await model_call.complete(route, _USER, timeout=30) + + assert (reply.content, reply.usage, reply.cost.status, reply.cost.unpriced) == ("", None, "unpriced", "no_tokens") + + +def test_a_url_elsewhere_needs_a_key_of_its_own(monkeypatch): + _configure(monkeypatch, "https://gw.test/v1") + + with pytest.raises(ConfigError, match=r"^judge 'j': other.test isn't the configured model endpoint.*judge_litellm"): + model_call.resolve_route("gpt-4.1-mini", api_key=None, api_base="https://other.test/v1", who="judge 'j'", + key_how=_HOW) + own = model_call.resolve_route("gpt-4.1-mini", api_key="secret:own", api_base="https://other.test/v1", who="j", + key_how=_HOW) + configured = model_call.resolve_route("claude-sonnet-4-6", api_key=None, api_base="https://GW.test/v1/", who="j", + key_how=_HOW) + + assert (own.call.api_key, own.call.api_base, own.endpoint_host) == ("own-key", "https://other.test/v1", "other.test") + assert (configured.call.api_key, configured.call.api_base) == ("configured-key", "https://GW.test") + assert (configured.model, configured.litellm_model) == ("claude-sonnet-4-6", "claude-sonnet-4-6") + + +def test_a_prefixed_name_with_no_endpoint_goes_by_its_provider(monkeypatch): + _configure(monkeypatch, None) + + route = model_call.resolve_route("openai/gpt-4o", api_key=None, api_base=None, who="j", key_how=_HOW) + + assert (route.litellm_model, route.call.api_base, route.endpoint_host) == ("openai/gpt-4o", None, None) + + +def _provider(name: str, custom_llm_provider: str | None = None): + providers = {"claude-sonnet-4-6": "anthropic", "gpt-4.1-mini": "openai", "gemini-2.5-flash": "vertex_ai", + "deepseek/deepseek-chat": "deepseek"} + if custom_llm_provider: + return name, custom_llm_provider, None, None + if name not in providers: + print("Provider List") + raise ValueError(name) + return name, providers[name], None, None + + +def test_the_route_keeps_a_native_name_and_sends_an_unknown_one_as_openai(capsys): + route = model_call.litellm_route + assert route("claude-sonnet-4-6", "https://gw.example/v1", _provider) == ("claude-sonnet-4-6", "https://gw.example") + assert route("gpt-4.1-mini", "https://gw.example/v1", _provider) == ("gpt-4.1-mini", "https://gw.example/v1") + assert route("qwen3:8b", "https://gw.example", _provider) == ("openai/qwen3:8b", "https://gw.example") + assert route("qwen3:8b", None, _provider) == ("qwen3:8b", None) + assert route("claude-sonnet-4-6", "https://gw.example/v1", _provider, "openai") == ( + "claude-sonnet-4-6", "https://gw.example/v1") + assert route("qwen3:8b", "https://gw.example", _provider, "hosted_vllm") == ("qwen3:8b", "https://gw.example") + assert route("gemini-2.5-flash", "https://gw.example/v1", _provider) == ( + "openai/gemini-2.5-flash", "https://gw.example/v1") + assert route("deepseek/deepseek-chat", "https://gw.example/v1", _provider) == ( + "deepseek/deepseek-chat", "https://gw.example/v1") + assert route("gemini-2.5-flash", "https://gw.example", _provider, "vertex_ai") == ( + "gemini-2.5-flash", "https://gw.example") + assert capsys.readouterr().out == "" + + +def test_setup_errors_are_the_ones_no_retry_fixes(): + setup = model_call.setup_errors() + + assert ConfigError in setup and litellm.AuthenticationError in setup and litellm.BudgetExceededError in setup + assert not issubclass(litellm.Timeout, setup) and not issubclass(litellm.RateLimitError, setup) diff --git a/tst/unit/config/test_one_model_call_path.py b/tst/unit/config/test_one_model_call_path.py new file mode 100644 index 000000000..5daad58e6 --- /dev/null +++ b/tst/unit/config/test_one_model_call_path.py @@ -0,0 +1,57 @@ +"""Core calls a model itself only through ``config/model_call.py``: no other module in ``src/agent_env`` imports +LiteLLM. Eval integrity and the record of each judge call's model, host and cost rest on no framework call going +around it. A shipped agent's folder (``agents/agents/``) isn't core: it runs in its own image with its own client.""" + +import ast +import pathlib + +import pytest + +SRC = pathlib.Path(__file__).resolve().parents[3] / "src" / "agent_env" +THE_PATH = "config/model_call.py" +NOT_CORE = ("agents/agents/",) + + +def _imports_litellm(tree: ast.AST) -> bool: + """Whether ``tree`` imports LiteLLM or a module of it, anywhere: by statement, or by ``import_module`` or + ``__import__`` with the name written out.""" + for node in ast.walk(tree): + if isinstance(node, ast.Import) and any(alias.name.split(".")[0] == "litellm" for alias in node.names): + return True + if isinstance(node, ast.ImportFrom) and not node.level and (node.module or "").split(".")[0] == "litellm": + return True + if (isinstance(node, ast.Call) and node.args and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) and node.args[0].value.split(".")[0] == "litellm" + and getattr(node.func, "attr", getattr(node.func, "id", None)) in {"import_module", "__import__"}): + return True + return False + + +def test_only_the_one_model_call_imports_litellm(): + offenders = sorted( + rel for path in SRC.rglob("*.py") + if (rel := path.relative_to(SRC).as_posix()) != THE_PATH and not rel.startswith(NOT_CORE) + and _imports_litellm(ast.parse(path.read_text())) + ) + + assert offenders == [], ( + f"{offenders} import LiteLLM. Core calls a model only through {THE_PATH} (route, complete), which records " + "what each call called and what it cost; call it from there.") + assert _imports_litellm(ast.parse((SRC / THE_PATH).read_text())) + + +@pytest.mark.parametrize("source", [ + "import litellm", "import litellm.cost_calculator as c", "from litellm import acompletion", + "from litellm.types.utils import ModelResponse", "def f():\n import litellm", "importlib.import_module('litellm')", + "__import__('litellm.utils')", +]) +def test_the_guard_sees_each_way_of_importing_litellm(source): + assert _imports_litellm(ast.parse(source)) + + +@pytest.mark.parametrize("source", [ + "import litellm_helper", "from .litellm import x", "from . import litellm", "litellm_api_key = 'k'", + "importlib.import_module(name)", +]) +def test_the_guard_leaves_other_names_alone(source): + assert not _imports_litellm(ast.parse(source)) diff --git a/tst/unit/config/test_trajectory_formats.py b/tst/unit/config/test_trajectory_formats.py new file mode 100644 index 000000000..24a6a40d6 --- /dev/null +++ b/tst/unit/config/test_trajectory_formats.py @@ -0,0 +1,28 @@ +"""The trajectory formats beside native a run prescribes: ``[trajectories] formats`` in config.toml, else none.""" + +import pytest + +from agent_env.config.errors import ConfigError +from tst.util.config import config_with_document + + +def test_a_run_prescribes_no_format_but_native_unless_config_names_some(): + assert config_with_document({}).get_trajectory_formats() == () + assert config_with_document({"trajectories": {}}).get_trajectory_formats() == () + + +def test_trajectories_formats_names_the_formats_a_run_prescribes(): + assert config_with_document({"trajectories": {"formats": ["atif/1"]}}).get_trajectory_formats() == ("atif/1",) + assert config_with_document({"trajectories": {"formats": []}}).get_trajectory_formats() == () + + +def test_trajectories_formats_resolves_env_references(monkeypatch): + monkeypatch.setenv("AGENTENV_TEST_TRAJECTORY_FORMAT", "atif/1") + formats = ["env:AGENTENV_TEST_TRAJECTORY_FORMAT", "env:AGENTENV_TEST_UNSET?other/2"] + assert config_with_document({"trajectories": {"formats": formats}}).get_trajectory_formats() == ("atif/1", "other/2") + + +@pytest.mark.parametrize("formats", ["atif/1", [1], [""], {"atif/1": True}]) +def test_formats_that_arent_a_list_of_names_are_refused(formats): + with pytest.raises(ConfigError, match=r"\[trajectories\] formats must be a list of trajectory format names"): + config_with_document({"trajectories": {"formats": formats}}).get_trajectory_formats() diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index ec1c805dd..f50062d48 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -98,9 +98,9 @@ async def main(): anyio.run(main) ''') -#: How long a REPL may sit unused before the driver closes it, in the idle test: well over two calls in a row on a busy -#: runner, so the second still finds the first's REPL. -IDLE_S = 1.0 +#: How long the idle test's REPL may sit unused: through its two calls, however slow a busy runner makes them, then for +#: a moment once they're done, so the sweep closes it. +IDLE_THROUGH_CALLS_S, IDLE_AFTER_CALLS_S = 60, 0.2 MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") WINDOWS = Platform(os="windows", version="11", arch="amd64", display="dwm") @@ -360,10 +360,11 @@ async def js(caller: str, code: str) -> str: @pytest.mark.asyncio async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): monkeypatch.setattr(module, "SWEEP_S", 0.05) - monkeypatch.setattr(module, "IDLE_S", IDLE_S) + monkeypatch.setattr(module, "IDLE_S", IDLE_THROUGH_CALLS_S) async with mcp_session(_driver(module, monkeypatch)) as session: await session.call_tool("js", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) before = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + monkeypatch.setattr(module, "IDLE_S", IDLE_AFTER_CALLS_S) closed = await _gone(int(before.rpartition(" in ")[2])) after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) diff --git a/tst/unit/env/envs/test_env_card_loads.py b/tst/unit/env/envs/test_env_card_loads.py index 37c385d18..533feb432 100644 --- a/tst/unit/env/envs/test_env_card_loads.py +++ b/tst/unit/env/envs/test_env_card_loads.py @@ -95,6 +95,25 @@ async def test_a_child_env_missing_from_the_card_takes_the_legacy_path(sent): assert json.loads(request.content) == {"mock_data_path": "/data/email.json"} +@pytest.mark.asyncio +@pytest.mark.parametrize("token, sent_as", [("s3cret", "Bearer s3cret"), (None, None)], ids=["token", "no-token"]) +async def test_the_legacy_path_sends_its_records_control_token_when_it_has_one(sent, token, sent_as): + # Neither email nor shop serves a card, so the gateway's card, read at its own address, lists neither. + deployed = DeployedGatewayEnv( + env_id="multi", env_version=1, gateway_url=_GW, mcp_url=f"{_GW}/mcp", db_web_url=None, sandbox_id="sb-1", + environment_card_url=f"{_GW}{WELL_KNOWN_PATH}", environment_card=await _card("mcp-slack"), control_token=token) + web = _website() + web._sandbox, web._gateway_url, web._deployed = MagicMock(), _GW, deployed + web._copy_artifact_into_container = AsyncMock(return_value="/tmp/data/shop.json") + web._env_provider = MagicMock(spec=EnvironmentGatewayProvider, install_changelog_triggers=AsyncMock()) + + await _mcp(deployed).load_environment_artifact(_artifact("email")) + await web.load_environment_artifact(_artifact("shop")) + + assert [(r.url.path, r.headers.get("authorization")) for r in sent] == [ + ("/svc/mcp-email/api/reset", sent_as), ("/svc/shop/api/reset", sent_as), ("/svc/shop/api/add", sent_as)] + + @pytest.mark.asyncio async def test_the_legacy_path_refuses_an_environment_of_several_files_before_sending_anything(sent): several = SimpleNamespace(environment_name="email", id="mail-seed", file_artifact_refs={"a.json": None, "b.json": None}) diff --git a/tst/unit/env/envs/test_mcp_server_plugin_provider.py b/tst/unit/env/envs/test_mcp_server_plugin_provider.py index b37528070..3959f27b4 100644 --- a/tst/unit/env/envs/test_mcp_server_plugin_provider.py +++ b/tst/unit/env/envs/test_mcp_server_plugin_provider.py @@ -16,7 +16,7 @@ from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI from agent_env.config import get_config, reset_config -from agent_env.env.env import DeployedEnv, DeployedSandboxEnv +from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, DeployedSandboxEnv from agent_env.env.envs._deployment import LOAD_OPERATIONS from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv @@ -228,9 +228,9 @@ async def test_close_closes_the_plugin_that_deployed(): async def test_a_multi_env_child_of_a_plugin_type_is_still_the_gateways(): child = _env() multi = MultiEnv(id="multi", version=1, mcp_server_envs=[child]) - record = SimpleNamespace(env_id="multi", env_version=1, sandbox_id="gw", sandbox_type="modal", sandbox_ids={}, - env_state_instance_ids=[], instance_id="i", mcp_server_name="m", environment_card=None, - environment_card_url=None, gateway_url="https://gw.example") + record = DeployedGatewayEnv(env_id="multi", env_version=1, sandbox_id="gw", sandbox_type="modal", instance_id="i", + mcp_server_name="m", gateway_url="https://gw.example", mcp_url="https://gw.example/mcp", + db_web_url=None) gw = MagicMock(sandbox_id="gw", mode="vm", type="modal") with patch("agent_env.env.env.Env.get", return_value=multi), \ patch("agent_env.providers.build_sandbox_provider", return_value=MagicMock(get_sandbox=AsyncMock(return_value=gw))), \ diff --git a/tst/unit/env/envs/test_server_reset.py b/tst/unit/env/envs/test_server_reset.py index 42adeec6b..6dfa13a96 100644 --- a/tst/unit/env/envs/test_server_reset.py +++ b/tst/unit/env/envs/test_server_reset.py @@ -95,6 +95,17 @@ async def test_an_older_server_is_emptied_through_its_api_reset_and_one_answerin assert [call.args for call in offers.await_args_list] == ([] if reset else [(f"{GATEWAY}/mcp", "email_reset")]) +@pytest.mark.asyncio +@pytest.mark.parametrize("token, sent", [("s3cret", "Bearer s3cret"), (None, None)], ids=["token", "no-token"]) +async def test_an_older_servers_api_reset_sends_its_records_control_token_when_it_has_one(server, token, sent): + env = _server(None) + env._deployed.control_token = token + with patch("agent_env.env.legacy_protocol.v1_base_url", AsyncMock(return_value=None)): + assert await _deployment.reset_server_data(env, mcp=True) is True + assert [(request.url.path, request.headers.get("authorization")) for request in server.requests] == [ + ("/svc/mcp-email/api/reset", sent)] + + @pytest.mark.asyncio async def test_an_older_server_answering_404_resets_through_its_tool_with_no_seed_to_the_data_it_boots_with(server): server.status = 404 diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index 6a9a22771..b968aaa54 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -1,5 +1,6 @@ -"""The gateway's control routes, the data plane and an MCP server's own endpoint behind its proxy take the deploy's -control token when it has one; every other route stays open.""" +"""The gateway's control routes, and behind its proxy the data plane, an MCP server's own endpoint and the older REST +routes that reset, load or export state, take the deploy's control token when it has one; every other route stays +open.""" from __future__ import annotations import asyncio @@ -12,7 +13,7 @@ from agent_env.env.gateway import InternalMCPServer from agent_env.env.gateway import rest_proxy as rest_proxy_module -from agent_env.env.gateway.constants import CONTROL_ROUTES, WELL_KNOWN_PATH +from agent_env.env.gateway.constants import CONTROL_ROUTES, LEGACY_STATE_ROUTES, WELL_KNOWN_PATH from agent_env.env.gateway.gateway import Gateway TOKEN = "s3cret" @@ -27,17 +28,37 @@ "/clock/set-time": "PUT", "/clock/advance": "PUT", "/clock/clear": "POST"} -# Behind the proxy: the data plane, its extensions, each service's included, and an MCP server's own endpoint, however -# the path is spelt on its way to the gateway. +LEGACY = {"/api/reset": "POST", "/api/add": "POST", "/api/bridge/apply": "POST", "/export-state": "GET", + "/export-snapshot": "GET", "/snapshot-dirty": "GET", "/reset-memory": "POST", "/validate-rows": "POST", + "/api/_state": "GET", "/api/_reset": "POST", "/api/debug/seed": "POST"} + +# Behind the proxy: the data plane, its extensions, each service's included, an MCP server's own endpoint, and the +# older REST routes on an MCP server's route and a site's, however the path is spelt on its way to the gateway. GUARDED = [("POST", "/agentenv"), ("POST", "/svc/mcp-svc/agentenv"), ("POST", "/svc/mcp-svc/agentenv/ext/grant"), ("POST", "/svc/mcp-svc/x/../agentenv"), ("POST", "/svc/mcp-svc/%61gentenv"), ("POST", "/svc/shop/agentenv"), ("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"), ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv"), ("POST", "/svc/mcp-svc/public%2Fx/../mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv/ext/set_acting_user"), - ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP")] + ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP"), + *((method, f"/svc/{key}{path}") for key in ("mcp-svc", "shop") for path, method in LEGACY.items()), + ("GET", "/svc/mcp-svc/export-snapshot?stream=1"), ("POST", "/svc/shop/api/reset/x"), + ("GET", "/svc/mcp-svc/x/../export-state"), ("GET", "/svc/mcp-svc/%65xport-state"), + ("POST", "/svc/mcp-svc/public%2Fx/../api/reset"), ("POST", "/svc/shop/public%2Fx/../api/add"), + ("POST", "/svc/mcp-svc/API/Reset"), ("GET", "/svc/shop/Export-Snapshot"), ("POST", "/svc/mcp-svc/api//reset"), + ("POST", "/svc/mcp-svc/api/bridge/../bridge/apply"), ("GET", "/svc/shop/api/_state/"), + ("GET", "/svc/shop/API/_State"), ("GET", "/svc/shop/x/../api/_state"), ("GET", "/svc/shop/api/%5Fstate")] PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"), - ("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), - ("GET", "/website/shop/index.html")] + ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), ("GET", "/website/shop/index.html"), + ("GET", "/svc/mcp-svc/export-statex"), ("POST", "/svc/mcp-svc/api/resetx"), + ("POST", "/svc/shop/api/address"), ("GET", "/svc/mcp-svc/x/export-state"), + ("POST", "/svc/mcp-svc/api/bridge"), ("GET", "/website/shop/export-statex"), + ("GET", "/svc/shop/api/_statex"), ("GET", "/svc/shop/api/state"), ("GET", "/website/shop/api/state"), + ("GET", "/website/shop/api/_statex"), ("GET", "/svc/shop/api/debug/inspect")] +# A site's route never reaches these on a gateway with a token, however they are spelt, with the token or without. +SITE_REFUSED = [*(f"/website/shop{path}" for path in LEGACY), "/website/shop/api/reset/x", + "/website/shop/export-snapshot?stream=1", "/website/shop/x/../export-state", + "/website/shop/x/../api/add", "/website/shop/public%2Fx/../api/reset", + "/website/shop/x/../api/_state", "/website/shop/api/%5Fstate"] _RealAsyncClient = httpx.AsyncClient @@ -48,6 +69,7 @@ def _client(token: str | None) -> httpx.AsyncClient: def test_the_routes_under_test_are_the_ones_the_gateway_covers(): assert set(CONTROL) == CONTROL_ROUTES + assert set(LEGACY) == set(LEGACY_STATE_ROUTES) @pytest.mark.asyncio @@ -151,7 +173,7 @@ async def _proxied(token: str | None, method: str, path: str, headers: dict[str, "website_urls": {"shop": "http://shop-backend"}, **children}) url = urlsplit(path) scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "http_version": "1.1", "method": method, - "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": b"", + "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": url.query.encode(), "root_path": "", "server": ("gateway", 80), "client": ("agent", 1), "headers": [(b"content-type", b"application/json"), *((name.lower().encode(), value.encode()) for name, value in headers.items())]} @@ -182,7 +204,8 @@ async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_refuse caplog.set_level(logging.WARNING, logger="agent_env.env.gateway.gateway") assert await _proxied(TOKEN, method, path, headers) == 401 assert backing.paths == [] - assert [r.getMessage() for r in caplog.records] == [f"refused {method} {unquote(path)}: it takes the env's control token"] + assert [r.getMessage() for r in caplog.records] == [ + f"refused {method} {unquote(urlsplit(path).path)}: it takes the env's control token"] @pytest.mark.asyncio @@ -208,6 +231,23 @@ async def test_a_gateway_started_without_a_token_guards_nothing_behind_its_proxy assert await _proxied(None, method, path, {}) != 401 +@pytest.mark.asyncio +@pytest.mark.parametrize("path", SITE_REFUSED) +@pytest.mark.parametrize("headers", [{}, {"Authorization": f"Bearer {TOKEN}"}], ids=["none", "token"]) +async def test_a_sites_route_never_reaches_its_reset_or_add_on_a_gateway_with_a_token(monkeypatch, path, headers): + backing = _Backing(monkeypatch) + assert await _proxied(TOKEN, "POST", path, headers) == 404 + assert [at for at in backing.paths if at != WELL_KNOWN_PATH] == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", SITE_REFUSED) +async def test_a_sites_route_passes_its_reset_and_add_on_a_gateway_without_a_token(monkeypatch, path): + backing = _Backing(monkeypatch) + assert await _proxied(None, "POST", path, {}) == 200 + assert [at for at in backing.paths if at != WELL_KNOWN_PATH] + + @pytest.mark.asyncio @pytest.mark.parametrize("method, path", [("POST", "/svc/mcp-x/rpc"), ("GET", "/svc/mcp-x/rpc/"), ("POST", "/svc/mcp-x/agentenv")]) async def test_an_mcp_server_whose_endpoint_is_elsewhere_is_guarded_at_its_endpoint(monkeypatch, method, path): diff --git a/tst/unit/env/gateway/website_reserved_paths_test.py b/tst/unit/env/gateway/website_reserved_paths_test.py index 40c238684..2a7967989 100644 --- a/tst/unit/env/gateway/website_reserved_paths_test.py +++ b/tst/unit/env/gateway/website_reserved_paths_test.py @@ -203,3 +203,27 @@ def site(request): assert (await _http(gw, "OPTIONS", "/website/api/items", headers=preflight))[0] == 200 assert (await _http(gw, "OPTIONS", "/website/api", headers=preflight))[0] == 200 assert ("OPTIONS", "/items") in reached and ("OPTIONS", "/") in reached + + +@pytest.mark.asyncio +async def test_a_site_behind_a_base_path_refuses_its_reserved_paths_when_a_path_climbs_out_of_the_base_and_back(monkeypatch): + reached = [] + card = {"name": "shop", "url": "/agentenv", + "additionalInterfaces": [{"url": "http://shop-website-backend:8000/base/mcp", "transport": "mcp"}]} + + def site(request): + reached.append(request.url.path) + if request.url.path == f"/base{WELL_KNOWN_PATH}": + return httpx.Response(200, json=card) + return httpx.Response(200, content=_streamed()) + + transport = httpx.MockTransport(site) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(_Changelog(), website_urls={"shop": "http://shop-website-backend:8000/base"}, + rest_proxy_urls={"shop": "http://shop-website-backend:8000/base"}) + + for path in ("../base/agentenv", "../base/mcp", "../base/.well-known/agent-env.json", "x/../../base/agentenv", + "..%2Fbase/agentenv"): + assert (await _http(gw, "POST", f"/website/shop/{path}", body=b"{}"))[0] == 404, path + assert (await _http(gw, "GET", "/website/shop/../base/orders"))[0] == 200 + assert reached == [f"/base{WELL_KNOWN_PATH}", "/base/orders"] diff --git a/tst/unit/env/test_env.py b/tst/unit/env/test_env.py index 03414a75f..8be1611ae 100644 --- a/tst/unit/env/test_env.py +++ b/tst/unit/env/test_env.py @@ -129,6 +129,10 @@ async def test_a_record_with_a_control_token_sends_it_on_each_extension_call_onl assert [deployed.control_headers_for(url) for url in (env, f"{env}/svc/clock/agentenv", f"{env}0/x", "https://override/trajectory")] == [ {"Authorization": "Bearer t0ken"}] * 2 + [{}] * 2 + prefixed = _record(composed, _BASES[2]) + prefixed.control_token = "t0ken" + assert [prefixed.control_headers_for(f"{prefixed.environment_url}{path}") for path in ( + "/svc/clock/../x", "/svc/clock/../../../sb-other/x")] == [{"Authorization": "Bearer t0ken"}, {}] assert DeployedEnv.from_dict(dataclasses.asdict(deployed)).control_token == "t0ken" assert _record(composed).control_headers == {} and _record(composed).control_headers_for(env) == {} diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index d426e3259..93ea92264 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -7,10 +7,11 @@ import httpx import pytest from agentenv_protocol import RPC_PATH, WELL_KNOWN_PATH, DataPart, uploaded_file_part +from agentenv_protocol import client as protocol_v1 from agentenv_protocol.client import GetDataResponse from agent_env.env import legacy_protocol -from agent_env.env.env import DeployedSandboxEnv +from agent_env.env.env import DeployedGatewayEnv, DeployedSandboxEnv @pytest.mark.asyncio @@ -44,7 +45,7 @@ async def test_reset_uses_rest_on_success(monkeypatch): result = await legacy_protocol.reset("http://gw", "email", "/data/seed.json") assert result == {"ok": True} - rest.assert_awaited_once_with("http://gw/svc/mcp-email", "/data/seed.json") + rest.assert_awaited_once_with("http://gw/svc/mcp-email", "/data/seed.json", headers=None) mcp.assert_not_called() @@ -61,6 +62,21 @@ async def test_reset_falls_back_to_mcp_tool(monkeypatch): mcp.assert_awaited_once_with("http://gw/mcp", "email", "/data/seed.json", max_retries=2) +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [401, 403]) +async def test_a_reset_refused_for_its_token_raises_instead_of_falling_back_to_the_mcp_tool(monkeypatch, status): + """A refused token is a dropped or wrong token; the MCP tool would reset the service without one and hide it.""" + seen = _authorizations(monkeypatch, status=status) + mcp = AsyncMock() + monkeypatch.setattr(legacy_protocol, "reset_via_mcp_tool", mcp) + + with pytest.raises(httpx.HTTPStatusError) as refused: + await legacy_protocol.reset("http://gw", "email", "/data/seed.json") + + assert refused.value.response.status_code == status and seen == [("/svc/mcp-email/api/reset", None)] + mcp.assert_not_called() + + def test_service_base_url_mcp_vs_website(): assert legacy_protocol.environment_base_url("http://gw", "email", mcp=True) == "http://gw/svc/mcp-email" assert legacy_protocol.environment_base_url("http://gw", "slack", mcp=False) == "http://gw/svc/slack" @@ -179,6 +195,80 @@ async def test_a_v1_services_state_is_the_data_it_answers_with_else_its_export_s assert ("GET /svc/mcp-slack/export-state" in asked) is reads_export_state +def _authorizations(monkeypatch, rpc_parts: list | None = None, status: int = 200) -> list[tuple[str, str | None]]: + """Each request sent, as its path and its Authorization header; data/get answers with ``rpc_parts``, and every + other route with ``status``.""" + seen, real = [], httpx.AsyncClient + + def handle(request: httpx.Request) -> httpx.Response: + seen.append((request.url.path, request.headers.get("authorization"))) + if request.url.path.endswith("/agentenv"): + return httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"parts": rpc_parts or []}}) + return httpx.Response(status, json={}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + return seen + + +_LEGACY_CALLS = { + "/svc/mcp-email/api/reset": lambda **kw: legacy_protocol.reset_via_rest("http://gw/svc/mcp-email", "/data/seed.json", **kw), + "/svc/mcp-email/api/add": lambda **kw: legacy_protocol.add_via_rest("http://gw/svc/mcp-email", "/data/seed.json", **kw), + "/svc/mcp-email/export-state": lambda **kw: legacy_protocol.export_state("http://gw", "email", **kw), +} +_GRANT = {"http://gw": protocol_v1.control_headers("granted")} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path, call", _LEGACY_CALLS.items(), ids=list(_LEGACY_CALLS)) +@pytest.mark.parametrize("grants, headers, sent", [ + ({}, None, None), + (_GRANT, None, "Bearer granted"), + ({"http://other": protocol_v1.control_headers("granted")}, None, None), + (_GRANT, protocol_v1.control_headers("own"), "Bearer own"), + (_GRANT, {}, None), +], ids=["no-grant", "granted", "granted-elsewhere", "own-headers", "own-none"]) +async def test_a_legacy_route_sends_the_headers_it_is_given_else_the_runs_grant(monkeypatch, path, call, grants, headers, sent): + seen = _authorizations(monkeypatch) + with protocol_v1.authorized(grants): + await call(headers=headers) + assert seen == [(path, sent)] + + +def _gateway_record(card: dict, token: str | None) -> DeployedGatewayEnv: + """A record whose card is read at its gateway's own address, as a built-in gateway's is.""" + return DeployedGatewayEnv(env_id="e", env_version=1, gateway_url="http://gw", mcp_url="http://gw/mcp", db_web_url=None, + sandbox_id="sb-1", environment_card_url=f"http://gw{WELL_KNOWN_PATH}", environment_card=card, + control_token=token) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token, sent", [("t0ken", "Bearer t0ken"), (None, None)], ids=["token", "no-token"]) +async def test_a_services_export_state_is_read_with_its_deployments_control_token(monkeypatch, token, sent): + """slack answers data/get with a bundle, so its state is read from its export-state; email serves no card, so the + gateway's card lists none and its state is read from its export-state behind the gateway.""" + card = {"name": "gw", "children_environments": [{"name": "slack", "url": "/svc/mcp-slack/agentenv"}]} + seen = _authorizations(monkeypatch, [uploaded_file_part("slack.zip", name="slack.zip").model_dump(mode="json")]) + deployed = _gateway_record(card, token) + + await legacy_protocol.service_state(deployed, "http://gw", "slack") + await legacy_protocol.service_state(deployed, "http://gw", "email") + + assert seen == [("/svc/mcp-slack/agentenv", sent), ("/svc/mcp-slack/export-state", sent), + ("/svc/mcp-email/export-state", sent)] + + +@pytest.mark.asyncio +async def test_a_services_export_state_behind_another_gateway_is_read_with_the_runs_grant(monkeypatch): + """The record's token covers its own gateway only; at an override's gateway the run's grant decides.""" + seen = _authorizations(monkeypatch) + deployed = _gateway_record({"name": "gw", "children_environments": []}, "t0ken") + + with protocol_v1.authorized({"http://override": protocol_v1.control_headers("granted")}): + await legacy_protocol.service_state(deployed, "http://override", "email") + + assert seen == [("/svc/mcp-email/export-state", "Bearer granted")] + + def test_definition_servers_are_an_envs_mcp_servers_then_its_websites(): from types import SimpleNamespace diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py index 9016cc6d0..9d860b0c2 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_host_ports_test.py @@ -110,6 +110,15 @@ async def test_tunnel_urls_stay_keyed_by_container_port(fake_ports): assert sandbox.tunnel_urls[port] == f"http://127.0.0.1:{sandbox.host_port(port)}" +@pytest.fixture(autouse=True) +def machine_temp(tmp_path, monkeypatch): + """The machine's /tmp, which holds the cursor: this test's own, so parallel workers don't share it.""" + folder = tmp_path / "machine-tmp" + folder.mkdir() + monkeypatch.setattr(local_sandbox_module, "_MACHINE_TMP", folder) + return folder + + @pytest.fixture def busy(monkeypatch): """Ports the stand-in bind probe finds in use; every other port is free.""" @@ -119,9 +128,7 @@ def busy(monkeypatch): def _last_handed_out(port: int | str) -> None: - cursor = state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR - cursor.parent.mkdir(parents=True, exist_ok=True) - cursor.write_text(str(port)) + local_sandbox_module._host_port_cursor().write_text(str(port)) def test_host_ports_are_handed_out_in_turn_skipping_those_in_use(busy): @@ -146,17 +153,46 @@ def test_a_missing_or_unreadable_cursor_starts_at_a_random_port(busy, monkeypatc assert local_sandbox_module._free_host_port() == 25000 -def test_the_first_port_of_a_state_root_does_not_follow_a_reseeded_random(busy, tmp_path, monkeypatch): - """A trainer reseeds ``random``; fresh state roots must still start apart in the range.""" +def test_the_first_port_on_a_machine_does_not_follow_a_reseeded_random(busy, tmp_path, monkeypatch): + """A trainer reseeds ``random``; fresh machines must still start apart in the range.""" starts = [] - for root in ("a", "b", "c"): - monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / root)) + for machine in ("a", "b", "c"): + (tmp_path / machine).mkdir() + monkeypatch.setattr(local_sandbox_module, "_MACHINE_TMP", tmp_path / machine) random.seed(0) starts.append(local_sandbox_module._free_host_port()) assert len(set(starts)) > 1 +def test_processes_with_their_own_state_roots_or_temp_folders_hand_out_host_ports_in_turn(busy, tmp_path, monkeypatch): + """As each xdist worker does: a port one drew isn't drawn by the next before docker binds it.""" + _last_handed_out(20100) + drawn = [] + for root in ("a", "b", "c"): + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / root)) + monkeypatch.setenv("TMPDIR", str(tmp_path / root)) + drawn.append(local_sandbox_module._free_host_port()) + + assert drawn == [20101, 20102, 20103] + + +def test_a_machine_tmp_it_cant_make_its_folder_in_leaves_the_cursor_in_the_state_root(busy, tmp_path, monkeypatch): + """A file where /tmp would be, so not even root makes the folder.""" + (tmp_path / "not-a-folder").write_text("") + monkeypatch.setattr(local_sandbox_module, "_MACHINE_TMP", tmp_path / "not-a-folder") + + assert local_sandbox_module._host_port_cursor() == state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR + + +def test_a_machine_folder_others_can_write_leaves_the_cursor_in_the_state_root(busy, machine_temp, monkeypatch): + folder = machine_temp / f"agentenv-{os.getuid()}" + folder.mkdir(mode=0o777) + folder.chmod(0o777) + + assert local_sandbox_module._host_port_cursor() == state_root() / "locks" / local_sandbox_module._HOST_PORT_CURSOR + + def test_concurrent_draws_never_share_a_host_port(busy): """Each draw opens the cursor afresh, so the lock serialises threads as it does processes.""" drawn: list[int] = [] diff --git a/tst/unit/store/routing_test.py b/tst/unit/store/routing_test.py index 2a73af7b0..e0a0d431c 100644 --- a/tst/unit/store/routing_test.py +++ b/tst/unit/store/routing_test.py @@ -10,6 +10,7 @@ import pytest from click.testing import CliRunner +from agent_env.a2a_agent import conversation_store from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.artifacts.file import FileArtifact @@ -171,6 +172,66 @@ def test_merged_reads_sort_page_and_count_across_both_stores(stores): assert router.count_distinct("tasks", project) == 6 +@pytest.mark.parametrize("task_id, namespace", [(LOCAL_TASK, "local"), (HUB_TASK, "hf"), ("registry-task", "configured")]) +@pytest.mark.parametrize("sort", [None, Sort()], ids=["implicit", "empty"]) +def test_unsorted_find_one_stops_at_the_first_namespace_match(stores, monkeypatch, task_id, namespace, sort): + router, configured, _ = stores + backends = {"configured": configured, **router.namespaces} + for name, store in backends.items(): + store.insert("agent_env_a2a_conversations", {"conversation_id": "conversation", "namespace": name}) + + def unexpected_read(*args, **kwargs): + pytest.fail("an unsorted match must not read a later namespace") + + for name, store in backends.items(): + if name != namespace: + monkeypatch.setattr(store, "find_one", unexpected_read) + with run_scope(task_id): + result = router.find_one("agent_env_a2a_conversations", Filter.of(conversation_id="conversation"), sort) + assert result["namespace"] == namespace + + +@pytest.mark.parametrize("found", [False, True]) +def test_unsorted_find_one_falls_back_after_a_local_miss(stores, found): + router, configured, local = stores + local.insert("agent_env_a2a_conversations", {"conversation_id": "other"}) + expected = {"conversation_id": "conversation"} if found else None + if found: + configured.insert("agent_env_a2a_conversations", expected) + + with run_scope(LOCAL_TASK): + assert router.find_one("agent_env_a2a_conversations", Filter.of(conversation_id="conversation")) == expected + + +@pytest.mark.parametrize("task_id", [LOCAL_TASK, "registry-task"]) +@pytest.mark.parametrize("descending", [False, True]) +def test_sorted_find_one_compares_matches_across_namespaces(stores, task_id, descending): + router, configured, local = stores + configured.insert("records", {"rank": 1}) + local.insert("records", {"rank": 2}) + router.namespaces["hf"].insert("records", {"rank": 3}) + + with run_scope(task_id): + result = router.find_one("records", Filter(), Sort.by("rank", descending=descending)) + assert result["rank"] == (3 if descending else 1) + + +def test_appending_a_local_conversation_turn_does_not_read_the_shared_store(stores, monkeypatch): + _, configured, _ = stores + + def unexpected_read(*args, **kwargs): + pytest.fail("appending a local conversation turn must not read the shared store") + + with run_scope(LOCAL_TASK): + conversation_store.create_conversation("conversation", "instance", "user", "solver") + monkeypatch.setattr(configured, "find_one", unexpected_read) + result = conversation_store.add_a2a_task("conversation", [{"kind": "text", "text": "hello"}], "turn", "user") + + assert result["messages"][0]["parts"] == [{"kind": "text", "text": "hello"}] + assert result["a2a_tasks"][0]["a2a_task_id"] == "turn" + assert result["a2a_tasks"][0]["input_message_idx"] == 0 + + def test_reads_never_create_the_local_store(stores): router, _, _ = stores router.find_one("envs", Filter.of(id=LOCAL_ENV)) diff --git a/tst/unit/task/lint_test.py b/tst/unit/task/lint_test.py new file mode 100644 index 000000000..0d1f1ee00 --- /dev/null +++ b/tst/unit/task/lint_test.py @@ -0,0 +1,542 @@ +"""agent-env task lint: each rule with a task that breaks it and one that doesn't, where its findings refuse a run, and +the bundles agent-env ships linting clean.""" + +import importlib +import json +from typing import ClassVar + +import pytest +from click.testing import CliRunner + +from agent_env.bundle import BundleError, BundleKind, parse_bundle +from agent_env.bundle.installed import CORE, installed_bundles +from agent_env.bundle.plan import plan_bundle +from agent_env.bundle.resolve import build_step, resolve_bundle +from agent_env.bundle.run import planned_tasks +from agent_env.cli import cli +from agent_env.eval import Eval +from agent_env.eval.eval import EvalTask +from agent_env.openenv import app +from agent_env.store.routing import namespace_routing +from agent_env.task import Task +from agent_env.task.lint import UNPINNED, Finding, Severity, lint, said, unpinned_summary +from agent_env.task_step.task_step import TaskStep +from agent_env.training import trl +from agent_env.training.nemo_gym import __main__ as nemo_cli, prepare +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( + AgentPromptResponseVerifierTaskStep, +) +from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( + RunContainerUnitTestsVerifierTaskStep, +) + +BOX = {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm"} +TESTS = {"id": "tests", "type": "verify_sandbox", "sandbox_name": "box", "verifier_id": "tests", + "criteria": [{"type": "probe_file_exists", "criterion": "has it", "paths": ["a.txt"]}]} +MORE = {**TESTS, "id": "more", "verifier_id": "more"} +FINAL = {"id": "final", "type": "aggregate_verifiers", "verifier_id": "final", "verifier_ids": ["tests", "more"]} +NOTES = {"id": "notes", "type": "load_artifact", "sandbox_name": "box", "artifact_id": "shared-notes"} +# A validator: it grades, under a key of its own, with no verifier_id. +ASSESS = {"id": "assess", "type": "verify_mcp_env_assessment", "env_id": "mcp", "prompt_id": "ask"} + + +def _task(*steps, **fields) -> Task: + return Task.from_dict({"id": "t", "version": 1, "steps": [dict(step) for step in steps], **fields}) + + +def _found(task, *args, **kwargs) -> list[tuple[str, str, str]]: + return [(f.severity.value, f.rule, f.text) for f in lint(task, *args, **kwargs)] + + +def test_a_field_its_step_doesnt_read_is_an_error_naming_the_nearest_one_it_does(): + steps = [BOX, {**TESTS, "min_passing_scor": 0.8}] + + assert _found(_task(BOX, TESTS), steps) == [( + "error", "unknown-field", "step 'tests': verify_sandbox doesn't read 'min_passing_scor' (did you mean " + "'min_passing_score'?), so it would be ignored")] + assert _found(_task(BOX, TESTS), [BOX, TESTS]) == [] + + +def test_a_verifier_config_that_fails_when_it_grades_is_an_error_and_one_a_run_skips_is_a_warning(): + no_paths = {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "has it"}]} + unknown = {**TESTS, "criteria": [{"type": "probe_file_smells", "criterion": "has it"}]} + + assert _found(_task(BOX, no_paths)) == [("error", "step-config", "step 'tests': criterion #0: probe_file_exists " + "needs paths, a non-empty list of strings, not None")] + assert _found(_task(BOX, unknown)) == [("warning", "step-config", "step 'tests': criterion #0: type " + "'probe_file_smells' isn't one verify_sandbox checks, so a run skips it; it " + "checks ['bash_cmd_succeeds', 'probe_dir_exists', 'probe_file_contains', " + "'probe_file_exists']")] + assert _found(_task(BOX, no_paths), config=False) == [] + + +@pytest.mark.parametrize(("criterion", "problem"), [ + ({"type": "response_shouts"}, "unknown type 'response_shouts'"), + ({"type": "response_contains"}, "response_contains needs non-empty needles"), + ({"type": "response_regex_present"}, "response_regex_present needs a non-empty pattern"), + ({"type": "response_regex_present", "pattern": "(unclosed"}, "pattern '(unclosed' doesn't compile"), + ({"type": "response_regex_present", "pattern": "x", "flags": ["SHOUT"]}, "flags ['SHOUT'] aren't re flags"), +]) +def test_an_agent_prompt_response_criterion_a_run_would_refuse_is_a_config_problem(criterion, problem): + step = AgentPromptResponseVerifierTaskStep(id="check", version=1, prompt_id="ask", criteria=[criterion]) + fine = AgentPromptResponseVerifierTaskStep(id="check", version=1, prompt_id="ask", criteria=[ + {"type": "response_contains", "needles": ["42"]}, + {"type": "response_regex_present", "pattern": "4[0-9]", "flags": ["IGNORECASE", "IGNORECASE"]}]) + + (found,) = step.config_problems() + assert found.startswith(f"criterion #0: {problem}") + assert fine.config_problems() == [] + + +@pytest.mark.parametrize(("criterion", "problem"), [ + (None, "must be an object, not NoneType"), + ({"type": ["response_contains"]}, "unknown type ['response_contains']"), + ({"type": "response_contains", "needles": "abc"}, "response_contains needs non-empty needles, a list of strings"), + ({"type": "response_contains", "needles": [7]}, "response_contains needs non-empty needles, a list of strings"), + ({"type": "response_regex_present", "pattern": 7}, "response_regex_present needs a non-empty pattern, a string"), + ({"type": "response_regex_present", "pattern": "x", "flags": "IGNORECASE"}, "flags must be a list of re flag"), +]) +def test_an_agent_prompt_response_criterion_of_the_wrong_type_is_a_config_problem(criterion, problem): + step = AgentPromptResponseVerifierTaskStep(id="check", version=1, prompt_id="ask", criteria=[criterion]) + + (found,) = step.config_problems() + assert found.startswith(f"criterion #0: {problem}") + + +@pytest.mark.parametrize(("criterion", "problem"), [ + ({"type": ["probe_file_exists"], "criterion": "x"}, "type must be a string, not ['probe_file_exists']"), + ({"type": "probe_file_exists", "criterion": "x", "paths": "a.txt"}, + "probe_file_exists needs paths, a non-empty list of strings, not 'a.txt'"), + ({"type": "probe_file_contains", "criterion": "x", "paths": ["a.txt"], "expected": 7}, + "expected must be a string, not 7"), + ({"type": "bash_cmd_succeeds", "criterion": "x", "bash_cmd": ["make"]}, + "bash_cmd_succeeds needs a bash_cmd, a non-empty string, not ['make']"), +]) +def test_a_verify_sandbox_criterion_of_the_wrong_type_is_a_config_problem_and_no_warning(criterion, problem): + step = build_step({**TESTS, "criteria": [criterion]}) + + assert (step.config_problems(), step.config_warnings()) == ([f"criterion #0: {problem}"], []) + + +@pytest.mark.parametrize(("timeout", "problems"), [ + ("300", ["timeout_sec must be a number of seconds, not '300'"]), + (None, ["timeout_sec must be a number of seconds, not None"]), + (True, ["timeout_sec must be a number of seconds, not True"]), + (1.5, []), +]) +def test_a_unit_test_timeout_is_a_number_of_seconds(timeout, problems): + tests = RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", + command="pytest", timeout_sec=timeout) + + assert tests.config_problems() == problems + + +def test_the_cli_names_each_value_of_the_wrong_type_in_a_task_file_and_exits_1(tmp_path): + path = tmp_path / "bad.json" + path.write_text(json.dumps([BOX, {**TESTS, "criteria": [None, {"type": "probe_file_exists", "paths": "a.txt"}]}, + {"id": "check", "type": "agent_prompt_response_verifier", "prompt_id": "ask", + "criteria": [{"type": "response_regex_present", "pattern": 7}]}])) + + result = CliRunner().invoke(cli, ["task", "lint", str(path)]) + + assert result.exit_code == 1, result.output + assert result.exception is None or isinstance(result.exception, SystemExit) + assert [line.split(": ", 1)[1] for line in result.output.splitlines() if line.startswith(" - ")] == [ + "step 'tests': criterion #0: must be an object, not NoneType", + "step 'tests': criterion #1: probe_file_exists needs paths, a non-empty list of strings, not 'a.txt'", + "step 'check': criterion #0: response_regex_present needs a non-empty pattern, a string, not 7"] + + +def test_a_value_of_the_wrong_type_a_verifier_would_trip_on_is_a_config_problem(): + criteria = [{"type": "probe_file_exists", "criterion": "has it", "paths": ["a.txt"]}, None, "probe"] + tests = RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", + command=["pytest"], verifier_id=["tests"]) + + assert _found(_task(BOX, {**TESTS, "criteria": criteria})) == [ + ("error", "step-config", "step 'tests': criterion #1: must be an object, not NoneType"), + ("error", "step-config", "step 'tests': criterion #2: must be an object, not str")] + assert tests.config_problems() == ["verifier_id must be a string, not list", "command must be a string, not list"] + assert tests.config_warnings() == [] + assert _found(Task(id="t", version=1, steps=[*_task(BOX).steps, tests])) == [ + ("error", "step-config", "step 'tests': verifier_id must be a string, not list"), + ("error", "step-config", "step 'tests': command must be a string, not list")] + + +def test_a_unit_test_run_that_tests_nothing_is_a_warning_and_a_negative_timeout_an_error(): + def tests(**kw): + return RunContainerUnitTestsVerifierTaskStep(id="tests", version=1, sandbox_name="b", container_name="c", + **{"command": "pytest", **kw}) + + assert tests(command=" ").config_warnings() == [ + "command is empty, so it exits 0 and the tests pass without running anything"] + assert tests(command="", reward_path="/r").config_warnings() == [] + assert tests(timeout_sec=0).config_warnings() == [ + "timeout_sec 0 means no time limit, so a hung test runs until the sandbox's own TTL"] + assert tests(timeout_sec=-1).config_problems() == ["timeout_sec -1 is negative; give the tests at least 1 s"] + assert (tests().config_problems(), tests().config_warnings()) == ([], []) + + +class _Legacy(TaskStep): + """A verifier that still writes its legacy entry itself, as agentenvhub-sdk's do until they move onto the base.""" + + type: ClassVar[str] = "legacy_lint_test" + entity_refs = () + + def __init__(self, id): + super().__init__(id, 1) + self.verifier_id = id + + async def execute(self, context): + return context + + +class _Outside(TaskStep): + """An outside step off the base with no verifier_id, as agentenvhub-sdk's run_lighthouse_audit, which writes its + entry under a verification_id: lint can't tell what it writes.""" + + type: ClassVar[str] = "outside_lint_test" + entity_refs = () + + def __init__(self, id): + super().__init__(id, 1) + + async def execute(self, context): + return context + + +def test_a_headline_that_grades_nothing_is_an_error_and_one_that_writes_no_named_grade_a_warning(): + built = _task(BOX, TESTS).steps + legacy = Task(id="t", version=1, steps=[*built, _Legacy("cua")], headline_verifiers=["cua"]) + outside = Task(id="t", version=1, steps=[*built, _Outside("audit")], headline_verifiers=["audit"]) + + assert ("error", "headline-not-a-verifier", + "headline_verifiers names 'box', a deploy_sandbox step, which grades nothing") in _found( + _task(BOX, TESTS, headline_verifiers=["box"])) + for task, (name, kind) in [(legacy, ("cua", "legacy_lint_test")), (outside, ("audit", "outside_lint_test")), + (_task(BOX, ASSESS, headline_verifiers=["assess"]), + ("assess", "verify_mcp_env_assessment"))]: + assert ("warning", "headline-without-grade", + f"headline_verifiers names {name!r}, a {kind} step that writes no named grade, so a run's result will " + "read it as ungraded") in _found(task) + assert _found(_task(BOX, TESTS, headline_verifiers=["tests"])) == [] + + +def test_an_aggregate_source_no_verifier_writes_is_an_error_unless_a_step_lint_cant_see_into_may_write_it(): + on_assessment = {**FINAL, "verifier_ids": ["tests", "mcp_tool_correctness"]} + + assert _found(_task(BOX, TESTS, FINAL, headline_verifiers=["final"])) == [( + "error", "aggregate-source", "step 'final': verifier_ids names ['more'], which no verifier in this task writes " + "(it has ['final', 'tests'])")] + assert ("warning", "aggregate-source", + "step 'final': verifier_ids names ['mcp_tool_correctness'], which no verifier in this task names as its " + "verifier_id; unless ['assess'] write it, under keys lint can't see, the aggregate fails when it grades" + ) in _found(_task(BOX, ASSESS, TESTS, on_assessment, headline_verifiers=["final"])) + outside = Task(id="t", version=1, steps=[*_task(BOX, TESTS, FINAL).steps, _Outside("audit")], + headline_verifiers=["final"]) + assert [f.severity for f in lint(outside) if f.rule == "aggregate-source"] == [Severity.WARNING] + assert _found(_task(BOX, TESTS, MORE, FINAL, headline_verifiers=["final"])) == [] + + +def test_a_store_entity_read_with_no_version_warns_and_a_pinned_or_local_one_doesnt(): + pinned = {**NOTES, "artifact_id": None, "artifacts": [{"id": "shared-notes", "version": 2}]} + local = {**NOTES, "artifact_id": "@local/~/triage/notes"} + + assert _found(_task(BOX, NOTES, TESTS)) == [( + "warning", "unpinned-reference", "step 'notes': artifacts[0].id names artifact 'shared-notes' with no version, " + "so each run takes its latest; set version to pin it")] + assert _found(_task(BOX, NOTES, TESTS), unpinned=False) == [] + assert _found(_task(BOX, pinned, TESTS)) == _found(_task(BOX, local, TESTS)) == [] + + +def test_a_task_that_grades_nothing_warns_and_several_grades_without_a_headline_name_the_covering_aggregate(): + assert _found(_task(BOX)) == [("warning", "no-verifier", "no step grades this task, so a run has no score")] + assert _found(_task(BOX, ASSESS)) == [] + assert _found(Task(id="t", version=1, steps=[*_task(BOX).steps, _Outside("audit")])) == [] + assert _found(_task(BOX, TESTS, MORE, FINAL)) == [( + "warning", "no-headlines", "3 verifiers and no headline_verifiers, so every grade is top-line and anything that " + "needs one number must name one; 'final' aggregates the others, so name it in " + "headline_verifiers")] + assert _found(_task(BOX, TESTS, MORE))[0][2].endswith("name the one that is its reward in headline_verifiers") + assert _found(_task(BOX, TESTS)) == [] + + +def test_errors_come_before_warnings(): + found = lint(_task(BOX, NOTES, TESTS, FINAL), [BOX, NOTES, TESTS, {**FINAL, "sources": ["tests"]}]) + + assert [f.severity for f in found] == sorted((f.severity for f in found), key=lambda s: s is not Severity.ERROR) + assert {f.rule for f in found} == {"unknown-field", "aggregate-source", "unpinned-reference", "no-headlines"} + + +def _shipped(): + """agent-env's own bundles that hold tasks; the rest (drivers, computers, browsers) have none to lint.""" + return {bundle.name: bundle for bundle in installed_bundles() + if bundle.dist == CORE and any((bundle.root / "tasks").glob("*.json"))} + + +# The shipped examples that show env triggers and no grading, which lint rightly says grade nothing. +_UNGRADED = {"advance-to-next-event", "next-env-event"} + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.parametrize("name", sorted(_shipped())) +def test_a_shipped_bundles_tasks_lint_clean(name): + bundle = _shipped()[name] + with namespace_routing(): + parsed = parse_bundle(bundle.root, id_root=bundle.id_root) + tasks = [entry.name for entry in parsed.entries if entry.kind is BundleKind.TASK] + plan = plan_bundle(resolve_bundle(parsed), tasks=tasks, installed=installed_bundles) + + expected = [f"tasks/{task}.json: warning: no step grades this task, so a run has no score" for task in tasks + ] if name in _UNGRADED else [] + assert sorted(f"{task}: {finding}" for task, finding in plan.lint_findings) == sorted(expected) + + +@pytest.fixture +def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + root = tmp_path / "triage" + (root / "tasks").mkdir(parents=True) + return root + + +def _write(root, name, body): + (root / "tasks" / f"{name}.json").write_text(json.dumps(body)) + + +def test_the_cli_lints_a_bundle_folder_refusing_its_errors_and_printing_its_warnings(bundle_dir): + _write(bundle_dir, "ok", [BOX, TESTS, MORE]) + warned = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir)]) + _write(bundle_dir, "bad", [BOX, {**TESTS, "min_passing_scor": 0.8}]) + refused = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir)]) + + assert warned.exit_code == 0, warned.output + assert "tasks/ok.json: warning: 2 verifiers and no headline_verifiers" in warned.output + assert warned.output.endswith("task lint: 1 warning\n") + assert refused.exit_code == 1 + assert ("tasks/bad.json: step 'tests': verify_sandbox doesn't read 'min_passing_scor' (did you mean " + "'min_passing_score'?)") in refused.output + + _write(bundle_dir, "bad", [BOX, {**TESTS, "id": ["tests"]}]) + unparsed = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir)]) + assert (unparsed.exit_code, unparsed.output.splitlines()[-1]) == ( + 1, " - tasks/bad.json: step #1's id is a JSON array; it must be a string") + + +def test_the_cli_lints_one_task_file_of_a_bundle_or_one_on_its_own(bundle_dir, tmp_path): + _write(bundle_dir, "ok", [BOX, TESTS]) + alone = tmp_path / "alone.json" + alone.write_text(json.dumps({"headline_verifiers": ["tests"], "steps": [BOX, NOTES, TESTS]})) + + in_bundle = CliRunner().invoke(cli, ["task", "lint", str(bundle_dir / "tasks/ok.json")]) + on_its_own = CliRunner().invoke(cli, ["task", "lint", str(alone)]) + + assert (in_bundle.exit_code, in_bundle.output) == (0, "task lint: no problems\n") + assert on_its_own.exit_code == 0, on_its_own.output + assert f"{alone}: warning: step 'notes': artifacts[0].id names artifact 'shared-notes'" in on_its_own.output + + +def test_the_cli_lints_a_stored_task(local_stores): + task = _task(BOX, TESTS, headline_verifiers=["box"]) + stored = Task.put(id="t", steps=task.steps, headline_verifiers=["box"]) + + result = CliRunner().invoke(cli, ["task", "lint", "--id", "t"]) + + assert result.exit_code == 1 + assert f"t v{stored.version}: headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" in ( + result.output) + + +def test_task_validate_fails_on_a_lint_error_and_prints_a_warning(local_stores): + Task.put(id="graded", steps=_task(BOX, NOTES, TESTS).steps) + Task.put(id="headless", steps=_task(BOX, TESTS).steps, headline_verifiers=["box"]) + + warned = CliRunner().invoke(cli, ["task", "validate", "--id", "graded"]) + refused = CliRunner().invoke(cli, ["task", "validate", "--id", "headless"]) + + assert warned.exit_code == 0, warned.output + assert "graded v1: warning: step 'notes': artifacts[0].id names artifact 'shared-notes'" in warned.output + assert refused.exit_code == 1 + assert " - headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" in refused.output + + +def test_eval_run_refuses_an_eval_whose_task_has_a_lint_error_before_any_run(local_stores, monkeypatch): + task = _task(BOX, TESTS, headline_verifiers=["box"]) + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id="t", task_version=1)]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: task)) + monkeypatch.setattr(Task, "run", lambda *a, **k: pytest.fail("a run started")) + + result = CliRunner().invoke(cli, ["eval", "run", "--id", "e"]) + + assert result.exit_code == 1, result.output + assert ("Task lint found 1 error(s), so no run started (--skip-lint runs anyway):\n - t v1: headline_verifiers " + "names 'box', a deploy_sandbox step, which grades nothing") in result.output + + +def test_eval_run_with_skip_lint_says_the_error_and_goes_on_to_run(local_stores, monkeypatch): + task = _task(BOX, TESTS, headline_verifiers=["box"]) + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id="t", task_version=1)]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: task)) + monkeypatch.setattr(importlib.import_module("agent_env.cli.eval.run"), "print_banner", + lambda: (_ for _ in ()).throw(SystemExit(7))) + + result = CliRunner().invoke(cli, ["eval", "run", "--id", "e", "--skip-lint"]) + + assert result.exit_code == 7, result.output + assert " t v1: error: headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" in result.output + + +def test_eval_run_with_skip_lint_runs_past_a_steps_config_problem_as_it_did_before_it_linted(local_stores, + monkeypatch): + task = _task(BOX, {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "x"}]}) + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id="t", task_version=1)]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: task)) + monkeypatch.setattr(importlib.import_module("agent_env.cli.eval.run"), "print_banner", + lambda: (_ for _ in ()).throw(SystemExit(7))) + + refused = CliRunner().invoke(cli, ["eval", "run", "--id", "e"]) + skipped = CliRunner().invoke(cli, ["eval", "run", "--id", "e", "--skip-lint"]) + + assert refused.exit_code == 1, refused.output + assert "so no run started (--skip-lint runs anyway):\n - t v1: step 'tests': criterion #0: probe_file_exists" in ( + refused.output) + assert skipped.exit_code == 7, skipped.output + assert " t v1: error: step 'tests': criterion #0: probe_file_exists needs paths" in skipped.output + + +class _Planned(Exception): + """What a stand-in for a planning call raises once it has recorded the arguments it was given.""" + + +def _recording(calls): + def record(*args, **kwargs): + calls.append(kwargs) + raise _Planned() + return record + + +def test_selecting_a_bundles_tasks_lints_none_of_them_for_its_caller_checks_those_it_keeps(bundle_dir): + _write(bundle_dir, "t", {"headline_verifiers": ["box"], "steps": [BOX, TESTS]}) + + with namespace_routing(): + assert [entry.entry.name for entry in planned_tasks(bundle_dir)] == ["t"] + with pytest.raises(BundleError, match="which grades nothing"): + plan_bundle(resolve_bundle(parse_bundle(bundle_dir)), tasks=["t"]) + + +@pytest.mark.parametrize("serves", ["build_app", "trl", "nemo_gym"]) +def test_each_way_a_trainer_serves_a_bundle_passes_skip_lint_to_its_plan(serves, bundle_dir, tmp_path, monkeypatch): + calls = [] + if serves == "build_app": + monkeypatch.setattr(app, "prepare_bundle", _recording(calls)) + serve = lambda skip: app.build_app(bundle_dir, skip_lint=skip) # noqa: E731 + elif serves == "trl": + monkeypatch.setattr(trl, "build_app", _recording(calls)) + serve = lambda skip: trl.AgentEnv.local(bundle_dir, skip_lint=skip) # noqa: E731 + else: + monkeypatch.setattr(prepare, "build_app", _recording(calls)) + serve = lambda skip: prepare.write_server(bundle_dir, tmp_path / "gym", skip_lint=skip) # noqa: E731 + + for skip in (False, True): + with pytest.raises(_Planned): + serve(skip) + + assert [call["skip_lint"] for call in calls] == [False, True] + + +def test_nemo_gyms_cli_takes_skip_lint_and_its_server_config_keeps_it(bundle_dir, tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(nemo_cli, "write_server", lambda *args, **kwargs: calls.append(kwargs) or tmp_path) + + result = CliRunner().invoke(nemo_cli.main, [str(bundle_dir), str(tmp_path / "gym"), "--skip-lint"]) + + assert result.exit_code == 0, result.output + assert calls[0]["skip_lint"] is True + + +def test_the_cli_takes_a_path_or_an_id_but_not_both(bundle_dir): + assert CliRunner().invoke(cli, ["task", "lint"]).exit_code == 2 + assert CliRunner().invoke(cli, ["task", "lint", str(bundle_dir), "--id", "t"]).exit_code == 2 + + +_HEADLESS = "headline_verifiers names 'box', a deploy_sandbox step, which grades nothing" + + +def test_a_run_of_a_bundle_with_a_lint_error_is_refused_unless_it_skips_lint_but_not_for_a_steps_config(bundle_dir): + _write(bundle_dir, "t", {"headline_verifiers": ["box"], "steps": [BOX, TESTS]}) + _write(bundle_dir, "u", [BOX, {**TESTS, "criteria": [{"type": "probe_file_exists", "criterion": "x"}]}]) + + refused = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "t", "--dry-run"]) + skipped = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "t", "--dry-run", "--skip-lint"]) + config = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "u", "--dry-run", "--skip-lint"]) + + assert refused.exit_code != 0 + assert f"tasks/t.json: {_HEADLESS}" in refused.output + assert "task lint found 1 of these; --skip-lint runs the tasks anyway" in refused.output + assert skipped.exit_code == 0, skipped.output + assert f"tasks/t.json: error: {_HEADLESS}" in skipped.output + assert "Would run:\n tasks/t.json v1\n" in skipped.output + assert config.exit_code != 0 + assert "tasks/u.json: step 'tests': criterion #0: probe_file_exists needs paths" in config.output + assert "--skip-lint" not in config.output + + +def test_bundle_register_with_skip_lint_registers_a_task_lint_refuses_saying_why_once(bundle_dir): + _write(bundle_dir, "t", {"headline_verifiers": ["box"], "steps": [BOX, TESTS]}) + + refused = CliRunner().invoke(cli, ["bundle", "register", str(bundle_dir), "--yes"]) + skipped = CliRunner().invoke(cli, ["bundle", "register", str(bundle_dir), "--yes", "--skip-lint"]) + + assert refused.exit_code != 0 + assert skipped.exit_code == 0, skipped.output + assert skipped.output.count(f"tasks/t.json: error: {_HEADLESS}") == 1 + assert Task.get("triage-t").headline_verifiers == ["box"] + + +def test_unpinned_references_are_said_as_one_line_counting_tasks_and_kinds(): + def unpinned(kind): + return Finding(Severity.WARNING, UNPINNED, f"names {kind} 'x' with no version", "s", kind) + + many = [(f"t{n} v1", unpinned("agent")) for n in range(3)] + [("t0 v1", unpinned("artifact")), + ("t1 v1", unpinned("artifact"))] + + assert unpinned_summary(many) == ( + "warning: 3 tasks name 5 store entities with no version, so each run takes the latest (agent 3, artifact 2); " + "agent-env task lint --id TASK lists a task's") + assert unpinned_summary(many[:1]).startswith("warning: 1 task names 1 store entity with no version") + assert unpinned_summary([("t v1", Finding(Severity.WARNING, "no-verifier", "grades nothing"))]) is None + + +def test_eval_run_says_its_tasks_unpinned_references_in_one_line_and_its_other_warnings_by_kind(local_stores, + monkeypatch): + tasks = {f"t{n}": Task.from_dict({"id": f"t{n}", "version": 1, "steps": [BOX, {**NOTES, "id": f"notes{n}"}]}) + for n in range(4)} + listed = Eval(id="e", version=1, tasks=[EvalTask(task_id=name, task_version=1) for name in tasks]) + monkeypatch.setattr(Eval, "get", classmethod(lambda cls, id, version=None: listed)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: tasks[id])) + monkeypatch.setattr(importlib.import_module("agent_env.cli.eval.run"), "print_banner", + lambda: (_ for _ in ()).throw(SystemExit(7))) + + result = CliRunner().invoke(cli, ["eval", "run", "--id", "e"]) + + assert result.exit_code == 7, result.output + assert (" warning: 4 tasks name 4 store entities with no version, so each run takes the latest (artifact 4); " + "agent-env task lint --id TASK lists a task's") in result.output + assert "names artifact 'shared-notes'" not in result.output + assert (" warning: no step grades this task, so a run has no score (4 tasks: t0 v1, t1 v1, t2 v1 and 1 more; " + "agent-env task lint lists each)") in result.output + + +def test_a_run_says_each_kind_of_finding_once_with_how_many_tasks_have_it_errors_first(): + doubt, other = (Finding(Severity.WARNING, "no-verifier", text) for text in ("grades nothing", "unpinned")) + wrong = Finding(Severity.ERROR, "step-config", "needs paths", "tests") + findings = [*((f"tasks/{n}.json", doubt) for n in "abcde"), ("tasks/a.json", other), ("tasks/c.json", wrong), + ("tasks/d.json", wrong)] + + assert said(findings) == [ + "error: step 'tests': needs paths (2 tasks: tasks/c.json, tasks/d.json; agent-env task lint lists each)", + "warning: grades nothing (5 tasks: tasks/a.json, tasks/b.json, tasks/c.json and 2 more; agent-env task lint " + "lists each)", + "tasks/a.json: warning: unpinned"] diff --git a/tst/unit/task_step/run_context_object_url_test.py b/tst/unit/task_step/run_context_object_url_test.py index 180a09b8e..b6f11b4ef 100644 --- a/tst/unit/task_step/run_context_object_url_test.py +++ b/tst/unit/task_step/run_context_object_url_test.py @@ -54,6 +54,22 @@ def test_the_stored_form_round_trips(): assert PromptResponse.from_dict(response.to_dict()) == response +def test_every_field_round_trips_through_the_stored_form(): + """``from_dict`` names each field by hand, so one it leaves out would be dropped from every stored run.""" + values = {"prompt_text": "do it", "agent_trajectory_file_path": "/t.json", + "source_agent_per_turn_prompt_parts": [None, [{"kind": "text", "text": "more"}]], "tool_call_count": 3, + "model": "m", "error_type": "agent_error", "error_code": "c", "error_message": "e", + "agent_session_id": "s", "a2a_context_id": "ctx", "agent_name": "solver", "step_id": "solve", + "structured_output": {"a": 1}, "usage": [{"role": "solver", "turns": 2}], + "target_agent_per_turn_trajectories": [{"native_format": "echo/1", "atif": None}, + {"native_format": None, "atif": None}]} + response = dataclasses.replace(_neutral(), **values) + unset = [f.name for f in dataclasses.fields(PromptResponse) + if getattr(response, f.name) == (f.default if f.default is not dataclasses.MISSING else None)] + assert unset == [] + assert PromptResponse.from_dict(response.to_dict()) == response + + def test_the_legacy_key_wins_when_the_two_disagree(): """Only a raw-doc writer outside the model can make them differ, and today's know only the legacy key.""" response = PromptResponse.from_dict({"prompt_id": "p1", "response": "done", "agent_trajectory_s3_uri": _URL, diff --git a/tst/unit/task_step/test_always_on_trajectory_formats.py b/tst/unit/task_step/test_always_on_trajectory_formats.py new file mode 100644 index 000000000..9fc0ce6d0 --- /dev/null +++ b/tst/unit/task_step/test_always_on_trajectory_formats.py @@ -0,0 +1,37 @@ +"""always_on prescribes the run's trajectory formats to the agent it prompts, once, before its first wake, so the agent +builds atif/1 for the run's conversation only when the run prescribes it.""" + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps import prompt_agent as pa +from tst.unit.task_step.always_on_harness import SOLVER, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("formats", [["atif/1"], None]) +async def test_always_on_prescribes_the_runs_formats_to_its_agent_before_its_first_wake(monkeypatch, tmp_path, formats): + happened: list[tuple] = [] + + async def prescribe(context, step_id, agent, a2a_url): + happened.append(("prescribe", step_id, agent.agent_name, a2a_url, pa.prescribed_formats(context))) + + def agent(n, texts, rig): + happened.append(("wake", n)) + rig.quiet() + return "" + + monkeypatch.setattr(ao, "prescribe_formats", prescribe) + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + context = rig.context() + if formats is not None: + context.metadata["user_overrides"] = {"trajectory_formats": formats} + + await rig.run(rig.step(), context) + + assert happened[0] == ("prescribe", "week", "solver", SOLVER, tuple(formats or ())) + assert [event for event in happened if event[0] == "prescribe"] == [happened[0]] + assert happened[1][0] == "wake" diff --git a/tst/unit/task_step/test_prompt_agent.py b/tst/unit/task_step/test_prompt_agent.py index 457848273..e64f7d260 100644 --- a/tst/unit/task_step/test_prompt_agent.py +++ b/tst/unit/task_step/test_prompt_agent.py @@ -6,6 +6,7 @@ import asyncio import json +from collections.abc import Callable from datetime import UTC, datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -24,7 +25,7 @@ from agent_env.a2a_agent.object_transfer import DEFAULT_TRAJECTORY_MAX_BYTES from agent_env.config import configure from agent_env.store.object_store import LocalFilesystemObjectStore -from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext from agent_env.task_step.snapshot_utils import agent_state_capture from agent_env.task_step.task_steps import prompt_agent as pa from agent_env.task_step.task_steps.prompt_agent import ( @@ -138,10 +139,9 @@ def put(key, *args, **kwargs): monkeypatch.setattr(pa, "get_config", lambda: config) monkeypatch.setattr(agent_state_capture, "get_config", lambda: config) - # unbound call: the inline branch never touches `self`, so we skip AWS-touching construction. - uri = asyncio.run(PromptAgentTaskStep._fetch_trajectory( - object(), "http://a", {"params": {"endpoint": "/t"}}, "SERVER-ID", "s3://b/traj/", - "CLIENT-ID", sandbox_type="local", headers={})) + uri = asyncio.run(pa.fetch_native_trajectory( + "http://a", {"params": {"endpoint": "/t"}}, "SERVER-ID", "s3://b/traj/", + "CLIENT-ID", sandbox_type="local", headers={})).url assert uri == "s3://b/traj/trajectory-CLIENT-ID.json" assert puts_on_loop == [False] @@ -209,8 +209,7 @@ def issue_write_grant(self, object_url, *, media_type, max_bytes, expires_in=Non } } uri = asyncio.run( - PromptAgentTaskStep._fetch_trajectory( - object(), + pa.fetch_native_trajectory( "https://agent.example.test", extension, "SERVER-ID", @@ -218,7 +217,7 @@ def issue_write_grant(self, object_url, *, media_type, max_bytes, expires_in=Non "CLIENT-ID", sandbox_type="local", headers={}, ) - ) + ).url assert uri == "s3://b/traj/trajectory-CLIENT-ID.json" assert grants_on_loop == [False] @@ -230,46 +229,178 @@ def issue_write_grant(self, object_url, *, media_type, max_bytes, expires_in=Non assert descriptor["write"]["kind"] == "http-put" -@pytest.mark.asyncio -async def test_sdk_agent_on_the_local_store_still_records_its_trajectory(monkeypatch, tmp_path): +def _trajectory_extension(**enabled) -> dict: + async def sdk_get(request: TaskTrajectoryRequest | TaskObjectTrajectoryRequest): + return {"trajectory": []} + + [extension] = build_registry( + object(), [enable(TRAJECTORY_V1, **enabled)], sdk_handlers={(TRAJECTORY_V1.uri, "get"): sdk_get} + ).card_extensions() + return extension + + +async def _one_turn(monkeypatch, tmp_path, extension: dict, answer: Callable[[dict], dict] + ) -> tuple[PromptResponse, list[dict], LocalFilesystemObjectStore]: + """One turn of a prompt step against an agent whose card carries ``extension`` and whose trajectory reads + ``answer`` gives each body: the step's prompt response, the read bodies it sent and the store it wrote to.""" store = LocalFilesystemObjectStore(str(tmp_path)) configure(object_store=store) for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): monkeypatch.setattr(pa.conversation_store, fn, lambda *a, **kw: None) - - async def sdk_get(request: TaskTrajectoryRequest | TaskObjectTrajectoryRequest): - return {"trajectory": []} - - extensions = build_registry( - object(), [enable(TRAJECTORY_V1)], sdk_handlers={(TRAJECTORY_V1.uri, "get"): sdk_get} - ).card_extensions() - trajectory_requests: list[dict] = [] + requests: list[dict] = [] def trajectory(request: httpx.Request) -> httpx.Response: assert request.url.path == "/ext/trajectory" - trajectory_requests.append(json.loads(request.content)) - return httpx.Response(200, json={"trajectory": [{"type": "echo", "output": "Echo: hi"}]}) + requests.append(json.loads(request.content)) + return httpx.Response(200, json=answer(requests[-1])) FakeA2AAgents({"http://agent.test": [{"kind": "text", "text": "Echo: hi"}]}, other=trajectory).serve(monkeypatch) context = TaskStepContext(instance_id="ti-1") context.deployed_agents.append(DeployedAgent( agent_name="solver", api_url="http://agent.test", a2a_url="http://agent.test", - a2a_card={"capabilities": {"extensions": extensions}}, + a2a_card={"capabilities": {"extensions": [extension]}}, )) step = PromptAgentTaskStep( id="solve", version=None, prompt="hi", agent_name="solver", poll_interval_seconds=0, trajectory_output_prefix=store.object_url("prompt_agent_trajectories"), ) + return (await step.execute(context)).prompt_responses[-1], requests, store + - result = await step.execute(context) +@pytest.mark.asyncio +async def test_sdk_agent_on_the_local_store_still_records_its_trajectory(monkeypatch, tmp_path): + response, requests, store = await _one_turn( + monkeypatch, tmp_path, _trajectory_extension(), + lambda body: {"trajectory": [{"type": "echo", "output": "Echo: hi"}]}) - uri = result.prompt_responses[-1].agent_trajectory_object_url - assert trajectory_requests == [{"task_id": "task-1"}] + uri = response.agent_trajectory_object_url + assert requests == [{"task_id": "task-1"}] + assert response.target_agent_per_turn_trajectories == [{"native_format": None, "atif": None}] assert store.get_object_key(uri).startswith("prompt_agent_trajectories/trajectory-") assert json.loads(store.get(uri)) == [{"type": "echo", "output": "Echo: hi"}] +@pytest.mark.asyncio +async def test_a_card_that_offers_formats_is_asked_for_its_native_one_whatever_its_default(monkeypatch, tmp_path): + """A card whose default is atif/1, as F10 flips agents to, still gives judges the native copy they read.""" + extension = _trajectory_extension(native="echo/1") + extension["params"].update(formats=["echo/1", "atif/1"], default="atif/1") + + response, requests, store = await _one_turn( + monkeypatch, tmp_path, extension, + lambda body: {"trajectory": [{"type": "echo"}], "format": body.get("format", "atif/1")}) + + assert requests == [{"task_id": "task-1", "format": "echo/1"}] + assert response.target_agent_per_turn_trajectories == [{"native_format": "echo/1", "atif": None}] + assert json.loads(store.get(response.agent_trajectory_object_url)) == [{"type": "echo"}] + + +def test_a_runs_override_names_the_formats_it_prescribes_over_config(monkeypatch): + monkeypatch.setattr(pa, "get_config", lambda: SimpleNamespace(get_trajectory_formats=lambda: ("from-config/1",))) + assert pa.prescribed_formats(TaskStepContext()) == ("from-config/1",) + overridden = TaskStepContext(metadata={"user_overrides": {"trajectory_formats": ["atif/1"]}}) + assert pa.prescribed_formats(overridden) == ("atif/1",) + assert pa.prescribed_formats(TaskStepContext(metadata={"user_overrides": {"trajectory_formats": []}})) == () + with pytest.raises(StepContextError, match="trajectory_formats must be a list of format names"): + pa.prescribed_formats(TaskStepContext(metadata={"user_overrides": {"trajectory_formats": "atif/1"}})) + + +def _prescribing_agent(*, method: bool = True) -> DeployedAgent: + methods = {"get": {"request": {}}, **({"prescribe": {"request": {}}} if method else {})} + extension = {"uri": TRAJECTORY_V1.uri, "params": { + "formats": ["echo/1", "atif/1"] if method else ["echo/1"], "native": "echo/1", "default": "echo/1", + "methods": methods}} + return DeployedAgent(agent_name="solver", api_url="http://agent.test", a2a_url="http://agent.test", + a2a_card={"capabilities": {"extensions": [extension]}}) + + +def _prescribing(*formats: str) -> TaskStepContext: + return TaskStepContext(metadata={"user_overrides": {"trajectory_formats": list(formats)}}) + + +@pytest.mark.asyncio +async def test_an_agent_is_prescribed_only_formats_its_card_offers_and_the_step_records_what_it_builds( + monkeypatch, caplog): + sent: list[tuple[str, dict]] = [] + + def answer(request: httpx.Request) -> httpx.Response: + sent.append((request.url.path, json.loads(request.content))) + return httpx.Response(200, json={"formats": ["echo/1", "atif/1"]}) + + FakeA2AAgents({}, other=answer).serve(monkeypatch) + context = _prescribing("atif/1", "atif1") + + await pa.prescribe_formats(context, "solve", _prescribing_agent(), "http://agent.test") + + assert sent == [("/ext/trajectory/prescribe", {"formats": ["atif/1"]})] + assert context.metadata[pa.TRAJECTORY_FORMATS_KEY] == {"solve": { + "agent": "solver", "prescribed": ["atif/1", "atif1"], "built": ["echo/1", "atif/1"]}} + assert "agent 'solver' can't build ['atif1'], which the run prescribes: it builds ['echo/1', 'atif/1']" in caplog.text + + +@pytest.mark.asyncio +async def test_a_run_that_prescribes_atif_to_an_agent_that_cant_build_it_says_so(monkeypatch, caplog): + FakeA2AAgents({}, other=lambda request: pytest.fail(f"unexpected request to {request.url}")).serve(monkeypatch) + context = _prescribing("atif/1") + + await pa.prescribe_formats(context, "solve", _prescribing_agent(method=False), "http://agent.test") + + assert context.metadata[pa.TRAJECTORY_FORMATS_KEY]["solve"] == { + "agent": "solver", "prescribed": ["atif/1"], "built": None} + assert "agent 'solver' can't build ['atif/1'], which the run prescribes: it builds ['echo/1']" in caplog.text + + +@pytest.mark.asyncio +async def test_an_agent_on_an_older_sdk_that_offers_atif_unasked_gets_no_call_and_no_warning(monkeypatch, caplog): + """Its card lists atif/1 without ``prescribe``: it builds atif/1 for every conversation, so it isn't warned of.""" + agent = _prescribing_agent(method=False) + agent.a2a_card["capabilities"]["extensions"][0]["params"]["formats"] = ["echo/1", "atif/1"] + FakeA2AAgents({}, other=lambda request: pytest.fail(f"unexpected request to {request.url}")).serve(monkeypatch) + context = _prescribing("atif/1") + + await pa.prescribe_formats(context, "solve", agent, "http://agent.test") + + assert context.metadata[pa.TRAJECTORY_FORMATS_KEY]["solve"]["built"] is None + assert "can't build" not in caplog.text + + +@pytest.mark.asyncio +async def test_a_prescription_the_agent_cant_be_reached_for_is_tried_again(monkeypatch): + monkeypatch.setattr(pa, "_PRESCRIBE_RETRY_SECONDS", 0) + tries: list[int] = [] + + def flaky(request: httpx.Request) -> httpx.Response: + tries.append(1) + if len(tries) < 3: + raise httpx.ConnectError("agent restarting", request=request) + return httpx.Response(200, json={"formats": ["echo/1", "atif/1"]}) + + FakeA2AAgents({}, other=flaky).serve(monkeypatch) + context = _prescribing("atif/1") + + await pa.prescribe_formats(context, "solve", _prescribing_agent(), "http://agent.test") + + assert len(tries) == 3 and context.metadata[pa.TRAJECTORY_FORMATS_KEY]["solve"]["built"] == ["echo/1", "atif/1"] + + +@pytest.mark.asyncio +async def test_a_prescription_that_fails_leaves_the_run_going_and_records_why(monkeypatch, caplog): + monkeypatch.setattr(pa, "_PRESCRIBE_RETRY_SECONDS", 0) + + def refuse(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("agent gone", request=request) + + FakeA2AAgents({}, other=refuse).serve(monkeypatch) + context = _prescribing("atif/1") + + await pa.prescribe_formats(context, "solve", _prescribing_agent(), "http://agent.test") + + record = context.metadata[pa.TRAJECTORY_FORMATS_KEY]["solve"] + assert (record["built"], record["error"]) == (None, "ConnectError: agent gone") + assert "prescribing trajectory formats ['atif/1'] failed, so it builds what it was last prescribed" in caplog.text + + def test_a_prompt_step_is_a_prompt_agent_or_a_subclass_whether_a_step_or_its_config(): class _Unregistered(PromptAgentTaskStep): type = "unregistered_prompt" diff --git a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py index 5eb2490d3..6bbaa8dbf 100644 --- a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py +++ b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py @@ -13,7 +13,8 @@ from agent_env.task_step.task_steps import always_on as ao from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps import prompt_agent as pa +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, StoredTrajectory from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, _published, _step as _deploy_step from tst.unit.task_step.test_prompt_agent_user_side import AGENT, HANG, HUMAN, USER, Peers, _stored, exchange, text @@ -236,11 +237,11 @@ async def test_each_user_sim_turn_records_its_trajectory_as_always_on_does(monke {"uri": A2AAgent.EXT_TRAJECTORY, "params": {}}]}} fetched = [] - async def fetch(self, url, ext, task_id, prefix, name, *, sandbox_type, headers, card=None): + async def fetch(url, ext, task_id, prefix, name, *, sandbox_type, headers, card=None): fetched.append((url, task_id, prefix, sandbox_type)) - return f"{prefix}{name}.json" + return StoredTrajectory(f"{prefix}{name}.json") - monkeypatch.setattr(PromptAgentTaskStep, "_fetch_trajectory", fetch) + monkeypatch.setattr(pa, "fetch_native_trajectory", fetch) step = _step() await step.execute(ctx) diff --git a/tst/unit/task_step/test_prompt_agent_live_trajectory.py b/tst/unit/task_step/test_prompt_agent_live_trajectory.py index 99e04ff6b..5fde34290 100644 --- a/tst/unit/task_step/test_prompt_agent_live_trajectory.py +++ b/tst/unit/task_step/test_prompt_agent_live_trajectory.py @@ -16,6 +16,7 @@ a2a_agent, enable, ) +from agentenv_protocol.atif import FORMAT as ATIF, Adapter, atif_adapter, native from agent_env.config import configure from agent_env.store.object_store import LocalFilesystemObjectStore @@ -29,12 +30,23 @@ _MORE = [{"type": "assistant", "step": 1}, {"type": "result", "result": "done"}] -def _agent(store: LocalFilesystemObjectStore, *, live: bool, wait_for_chunk: bool) -> AgentEnvAgent: - """Appends two events, optionally waits until a chunk of them is stored, then two more.""" +@atif_adapter(native_format="test-events/1", version="1", input_includes_cache=False) +class _Events(Adapter): + @native(type="assistant") + def assistant(self, record: dict, turn) -> None: + turn.message(step=f"s{record['step']}", content=[f"step {record['step']}"]) + turn.step(key=f"s{record['step']}", llm_call_count=1) + + +def _agent(store: LocalFilesystemObjectStore, *, live: bool, wait_for_chunk: bool, native: str | None = None, + atif: bool = False) -> AgentEnvAgent: + """Appends two events, optionally waits until a chunk of them is stored, then two more; with ``atif``, it can + build atif/1 copies of them.""" + formats = {"native": native, **({"atif": _Events} if atif else {})} if native else {} @a2a_agent( identity=AgentIdentity(name="solver", description="test", version="1"), - extensions=(enable(TRAJECTORY_V1, live=live),), + extensions=(enable(TRAJECTORY_V1, live=live, **formats),), ) class Agent(AgentEnvAgent): async def run(self, request: TaskRequest) -> TaskResult: @@ -57,22 +69,29 @@ def _chunk_keys(store: LocalFilesystemObjectStore) -> list[str]: class _Run: - def __init__(self, monkeypatch: pytest.MonkeyPatch, tmp_path, *, live: bool, wait_for_chunk: bool) -> None: + def __init__(self, monkeypatch: pytest.MonkeyPatch, tmp_path, *, live: bool, wait_for_chunk: bool, + native: str | None = None, atif: bool = False) -> None: self.store = LocalFilesystemObjectStore(str(tmp_path)) configure(object_store=self.store) for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): monkeypatch.setattr(pa.conversation_store, fn, lambda *a, **kw: None) - app = _agent(self.store, live=live, wait_for_chunk=wait_for_chunk).create_app() + app = _agent(self.store, live=live, wait_for_chunk=wait_for_chunk, native=native, atif=atif).create_app() self.card = app.state.agentenv_a2a.card.model_dump(mode="json", exclude_none=True) self.cursor_reads: list[dict] = [] + self.final_reads: list[dict] = [] + #: Every request the step sent, in order: its path and, for a JSON-RPC call, its method. + self.sent: list[str] = [] + self.prescriptions: list[dict] = [] transport = httpx.ASGITransport(app=app) real_client = httpx.AsyncClient async def record(request: httpx.Request) -> None: + body = json.loads(await request.aread()) if request.method == "POST" else {} + self.sent.append(body.get("method", request.url.path) if isinstance(body, dict) else request.url.path) if request.url.path == "/ext/trajectory": - body = json.loads(await request.aread()) - if "after" in body: - self.cursor_reads.append(body) + (self.cursor_reads if "after" in body else self.final_reads).append(body) + elif request.url.path == "/ext/trajectory/prescribe": + self.prescriptions.append(body) monkeypatch.setattr( httpx, @@ -94,6 +113,50 @@ def step(self, **kwargs) -> PromptAgentTaskStep: ) +@pytest.mark.asyncio +@pytest.mark.parametrize("formats", [[ATIF], None]) +async def test_a_prompt_step_prescribes_the_runs_formats_before_its_first_message(monkeypatch, tmp_path, formats): + """So the agent builds atif/1 for the step's conversation only when the run prescribes it.""" + run = _Run(monkeypatch, tmp_path, live=False, wait_for_chunk=False, native="test-events/1", atif=True) + context = TaskStepContext(instance_id="ti-1") + if formats is not None: + context.metadata["user_overrides"] = {"trajectory_formats": formats} + + await run.execute(run.step(), context) + + assert run.prescriptions == [{"formats": formats or []}] + assert run.sent.index("/ext/trajectory/prescribe") < run.sent.index("message/send") + assert context.metadata["trajectory_formats"]["solve"]["built"] == ["test-events/1", *(formats or [])] + [final] = run.final_reads + async with httpx.AsyncClient() as client: + read = await client.post("http://agent.test/ext/trajectory", json={"task_id": final["task_id"], "format": ATIF}) + assert read.status_code == (200 if formats else 409) + + +@pytest.mark.asyncio +async def test_a_prompt_step_prescribes_nothing_to_an_agent_whose_card_offers_no_prescription(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path, live=False, wait_for_chunk=False, native="test-events/1") + context = TaskStepContext(instance_id="ti-1", metadata={"user_overrides": {"trajectory_formats": [ATIF]}}) + + await run.execute(run.step(), context) + + assert run.prescriptions == [] and "/ext/trajectory/prescribe" not in run.sent + + +@pytest.mark.asyncio +@pytest.mark.parametrize("native", ["test-events/1", None]) +async def test_every_read_names_the_native_format_a_card_offers_and_the_turn_records_the_label_served( + monkeypatch, tmp_path, native): + run = _Run(monkeypatch, tmp_path, live=True, wait_for_chunk=False, native=native) + + result = await run.execute(run.step(live_trajectory=True)) + + assert run.cursor_reads and {read.get("format") for read in run.cursor_reads} == {native} + assert [read.get("format") for read in run.final_reads] == [native] + assert result.prompt_responses[-1].target_agent_per_turn_trajectories == [ + {"native_format": "test-events/1", "atif": None}] + + @pytest.mark.asyncio async def test_a_live_turn_is_stored_in_chunks_while_it_runs(monkeypatch, tmp_path): run = _Run(monkeypatch, tmp_path, live=True, wait_for_chunk=True) diff --git a/tst/unit/task_step/test_prompt_agent_snapshots.py b/tst/unit/task_step/test_prompt_agent_snapshots.py index 879c6c1e4..e05c8becf 100644 --- a/tst/unit/task_step/test_prompt_agent_snapshots.py +++ b/tst/unit/task_step/test_prompt_agent_snapshots.py @@ -1296,6 +1296,9 @@ def get_default_human_a2a_url(self): def resolve_model(self, role, **layers): return resolve_model(role, **layers) + def get_trajectory_formats(self): + return () + monkeypatch.setattr(pa_mod, "get_config", lambda: _Cfg()) def context(self) -> TaskStepContext: diff --git a/tst/unit/task_step/test_recorded_models.py b/tst/unit/task_step/test_recorded_models.py index 792dbc9e0..ddb0db813 100644 --- a/tst/unit/task_step/test_recorded_models.py +++ b/tst/unit/task_step/test_recorded_models.py @@ -1,18 +1,22 @@ """A solver, user simulator or agent judge records the model its agent took: the one asked for when the agent's -card accepts a model, and None, the agent's own, when it doesn't.""" +card accepts a model, and None, the agent's own, when it doesn't. A direct judge records the model it called.""" +import json +from types import SimpleNamespace from unittest.mock import MagicMock +import litellm import pytest from a2a.types import TaskState from agent_env.a2a_agent import A2AAgent from agent_env.a2a_agent import protocol from agent_env.config.model import resolve_model -from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext from agent_env.task_step.task_steps import prompt_agent as pa_mod from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.util.config import install_model_config def _card(*supported: str) -> dict: @@ -100,3 +104,26 @@ async def post_agent_config(url, fields, headers=None): judge_effort=None, judge_max_thinking_tokens=None, context=context, headers={}) assert context.metadata["models"] == {"judge": {"judge": recorded}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("asked", "sent"), [("gpt-4.1-mini", "gpt-4.1-mini"), ("qwen3:8b", "openai/qwen3:8b")]) +async def test_a_direct_judge_records_the_model_it_called(monkeypatch, asked, sent): + install_model_config(monkeypatch, base_url="https://gw.test/v1", api_key="k") + called = [] + + async def acompletion(**kwargs): + called.append(kwargs["model"]) + answer = json.dumps({"results": [{"id": "c1", "score": 1.0, "justification": "it said hi"}]}) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=answer))]) + + monkeypatch.setattr(litellm, "acompletion", acompletion) + judge = RubricsVerifierTaskStep.from_dict({"id": "judge", "type": "rubrics_verifier", "prompt_id": "p", + "use_agent_judge": False, "use_trajectory": False, + "criteria": [{"id": "c1", "description": "says hi"}]}) + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p", response="hi", prompt_text="greet")], + metadata={"user_overrides": {"judge_model": asked}}) + + await judge.execute(context) + + assert called == [sent] and context.metadata["models"] == {"judge": {"judge": asked}} diff --git a/tst/unit/task_step/test_snapshot_agent_state.py b/tst/unit/task_step/test_snapshot_agent_state.py index 551f1cd26..71f6384b0 100644 --- a/tst/unit/task_step/test_snapshot_agent_state.py +++ b/tst/unit/task_step/test_snapshot_agent_state.py @@ -107,7 +107,7 @@ async def _supports_v1(_base): monkeypatch.setattr(protocol_v1, "supports_v1", _supports_v1) - async def _export_state(_gw, name): + async def _export_state(_gw, name, headers=None): return {"service": name, "rows": 1} monkeypatch.setattr(legacy_protocol, "export_state", _export_state) @@ -139,7 +139,7 @@ def put(self, *a, **kw): async def _no_v1(_base): return False - async def _export_state(_gw, name): + async def _export_state(_gw, name, headers=None): return {"service": name} monkeypatch.setattr(protocol_v1, "supports_v1", _no_v1) @@ -171,7 +171,7 @@ async def _get_data(base, timeout=30, headers=None): got.append(base) return GetDataResponse(parts=[DataPart(data={"rows": 1})]) - async def _export_state(gw, name): + async def _export_state(gw, name, headers=None): legacy.append((gw, name)) return {"rows": 1} @@ -223,6 +223,40 @@ def handle(request: httpx.Request) -> httpx.Response: assert set(json.loads(ctx.metadata["snapshot_json_url"])) == {"calendar", "contacts"} +@pytest.mark.asyncio +@pytest.mark.parametrize("token, sent", [("t0ken", "Bearer t0ken"), (None, None)], ids=["token", "no-token"]) +async def test_each_services_state_is_read_with_the_records_control_token(monkeypatch, token, sent): + """calendar answers data/get with a bundle, so its state comes from its export-state; contacts serves no card, so + its state comes from its export-state behind the gateway.""" + import httpx + from agentenv_protocol import uploaded_file_part + + from agent_env.env.gateway.constants import WELL_KNOWN_PATH + + store = _Store() + monkeypatch.setattr(artifact_mod, "EnvironmentUniverseArtifact", type("U", (), {"get": staticmethod(lambda _id: _StubUniverse())})) + monkeypatch.setattr(config_mod, "get_config", lambda: type("Cfg", (), {"get_object_store_at": lambda self, object_url: store})()) + seen, real = [], httpx.AsyncClient + bundle = uploaded_file_part("calendar.zip", name="calendar.zip").model_dump(mode="json") + + def handle(request: httpx.Request) -> httpx.Response: + seen.append((request.url.path, request.headers.get("authorization"))) + if request.url.path.endswith("/agentenv"): + return httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"parts": [bundle]}}) + return httpx.Response(200, json={"rows": 1}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + ctx = _ctx() + record = ctx.deployed_envs[0] + record.environment_card_url, record.environment_card = f"https://gw{WELL_KNOWN_PATH}", await _card("mcp-calendar") + record.control_token = token + + await _step()._capture_universe_state(ctx, _PREFIX) + + assert seen == [("/svc/mcp-calendar/agentenv", sent), ("/svc/mcp-calendar/export-state", sent), + ("/svc/mcp-contacts/export-state", sent)] + + async def _card(*keys: str) -> dict: """The real gateway's composed card over child envs at the given gateway keys (`mcp-{name}` or `{name}`).""" from agent_env.env.gateway.gateway import Gateway diff --git a/tst/unit/task_step/test_snapshot_env.py b/tst/unit/task_step/test_snapshot_env.py index cf75fb7ca..7419c0ddd 100644 --- a/tst/unit/task_step/test_snapshot_env.py +++ b/tst/unit/task_step/test_snapshot_env.py @@ -1043,6 +1043,69 @@ async def test_export_of_a_child_env_missing_from_the_card_streams_legacy_export assert [(r.method, str(r.url)) for r in sent] == [("GET", "https://gw-1/svc/mcp-gdrive/export-state")] +@pytest.mark.asyncio +@pytest.mark.parametrize("token, sent_as", [("t0ken", "Bearer t0ken"), (None, None)], ids=["token", "no-token"]) +async def test_export_streams_export_snapshot_and_export_state_with_the_records_control_token(monkeypatch, tmp_path, token, sent_as): + """gdrive streams its bundle from its export-snapshot; slack serves no card, so its state streams from its export-state.""" + from agent_env.task_step.task_steps import snapshot_env as mod + + record = DeployedGatewayEnv( + env_id="env-1", env_version=3, gateway_url="https://gw-1", mcp_url="https://gw-1/mcp", db_web_url=None, + sandbox_id="sb-1", environment_card_url=f"https://gw-1{WELL_KNOWN_PATH}", + environment_card=await _card({"mcp-gdrive": []}), control_token=token) + sent = _mock_http(monkeypatch, body=b"{}", rpc_result={"parts": [ + {"kind": "file", "file": {"uri": "export-snapshot?stream=1", "name": "gdrive.zip"}}]}) + + for name in ("gdrive", "slack"): + await mod.SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", name, str(tmp_path / name), 30, record) + + assert [(r.method, str(r.url), r.headers.get("authorization")) for r in sent] == [ + ("POST", "https://gw-1/svc/mcp-gdrive/agentenv", sent_as), + ("GET", "https://gw-1/svc/mcp-gdrive/export-snapshot?stream=1", sent_as), + ("GET", "https://gw-1/svc/mcp-slack/export-state", sent_as), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("uri, fetched, sent_as", [ + ("export-snapshot?stream=1", "https://proxy/sb/A/svc/mcp-gdrive/export-snapshot?stream=1", "Bearer t0ken"), + ("../../../B/collect", "https://proxy/sb/B/collect", None), + ("https://proxy/sb/B/collect", "https://proxy/sb/B/collect", None), +], ids=["own-service", "dotted-out", "same-host-elsewhere"]) +async def test_a_streamed_bundle_gets_the_token_only_at_its_own_service(monkeypatch, tmp_path, uri, fetched, sent_as): + """A service names the url its bundle streams from; behind a path-prefixed proxy another env shares the host.""" + from agent_env.task_step.task_steps import snapshot_env as mod + + record = DeployedGatewayEnv( + env_id="env-1", env_version=3, gateway_url="https://proxy/sb/A", mcp_url="https://proxy/sb/A/mcp", db_web_url=None, + sandbox_id="sb-1", environment_card_url=f"https://proxy/sb/A{WELL_KNOWN_PATH}", + environment_card=await _card({"mcp-gdrive": []}), control_token="t0ken") + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "file", "file": {"uri": uri, "name": "gdrive.zip"}}]}) + + with protocol_v1.authorized({"https://proxy/sb/B": protocol_v1.control_headers("other")}): + await mod.SnapshotEnvTaskStep._export_environment_to_file("https://proxy/sb/A", "gdrive", str(tmp_path / "g"), 30, record) + + assert [(str(r.url), r.headers.get("authorization")) for r in sent[1:]] == [(fetched, sent_as)] + + +@pytest.mark.asyncio +async def test_a_legacy_export_state_behind_another_gateway_is_read_with_the_runs_grant(monkeypatch, tmp_path): + """The record's token covers its own gateway only; at an override's gateway the run's grant decides.""" + from agent_env.task_step.task_steps import snapshot_env as mod + + record = DeployedGatewayEnv( + env_id="env-1", env_version=3, gateway_url="https://gw-1", mcp_url="https://gw-1/mcp", db_web_url=None, + sandbox_id="sb-1", environment_card_url=f"https://gw-1{WELL_KNOWN_PATH}", + environment_card={"name": "gw", "children_environments": []}, control_token="t0ken") + sent = _mock_http(monkeypatch, body=b"{}") + + with protocol_v1.authorized({"https://override": protocol_v1.control_headers("granted")}): + await mod.SnapshotEnvTaskStep._export_environment_to_file("https://override", "slack", str(tmp_path / "s"), 30, record) + + assert [(str(r.url), r.headers.get("authorization")) for r in sent if r.url.path.endswith("/export-state")] == [ + ("https://override/svc/mcp-slack/export-state", "Bearer granted")] + + @pytest.mark.asyncio async def test_snapshot_uses_the_stored_card_only_for_the_deployments_own_gateway(monkeypatch): """An override's gateway has its card read live, and the record's stored card describes its own gateway only.""" diff --git a/tst/unit/task_step/verifiers/test_judge_inputs.py b/tst/unit/task_step/verifiers/test_judge_inputs.py new file mode 100644 index 000000000..a1c11247d --- /dev/null +++ b/tst/unit/task_step/verifiers/test_judge_inputs.py @@ -0,0 +1,350 @@ +"""Every built-in judge's input, pinned: what a model judge is sent (its prompt, images, system prompt and cached +prefix), what an agent judge is given (its prompt and every file staged in its sandbox) and what a scripted verify is +called with, for runs whose native trajectories are stored as a run stores them. Each is what its golden file holds, +recorded before prompt steps kept per-turn trajectory records, and stays so byte for byte when the run's prompt +responses carry those records, read back from the stored form. Re-record after a deliberate change with +``AGENTENV_TEST_RECORD_GOLDEN=1`` and review the file's diff. Several cases build their runs with other tests' setups +(the always_on rig, user_sim_judge's fakes, the rubrics golden test's and run_context's helpers), so a change to one of +those is re-recorded here too.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import re +from collections.abc import Awaitable, Callable +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.config import get_config +from agent_env.env.env import DeployedEnv +from agent_env.providers.sandbox_providers import sandbox_provider +from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers import always_on_agent_verifier +from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import TrajectoryFilter +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.unit.task_step import test_always_on_agent_verifier as A +from tst.unit.task_step.verifiers import test_rubrics_verifier_golden as G +from tst.unit.task_step.verifiers import test_run_context as R +from tst.unit.task_step.verifiers import test_user_sim_judge as T +from tst.util import verifier_golden + +pytestmark = pytest.mark.usefixtures("local_stores") + +FILE = "judge_inputs.json" +RECORDED = Path(__file__).parent / "recorded" +_TURN = json.loads((RECORDED / "turn_claude_code.json").read_text()) +#: Two turns of a Claude Code run: the recorded turn's first half, then all of it. +TURNS = [json.dumps(_TURN[: len(_TURN) // 2]), json.dumps(_TURN)] + + +#: How many lines a long tool output has: enough for the default trajectory filter to compact it. +_LONG_OUTPUT_LINES = 400 + + +def _tool_span(n: int) -> dict: + output = f"line {n}\n" * _LONG_OUTPUT_LINES if n % 2 else "ok" + return {"name": f"tool_{n}", "attributes": {"gen_ai.operation.name": "execute_tool", + "gen_ai.prompt": json.dumps({"tool": f"tool_{n}", "input": {"n": n}}), + "gen_ai.completion": json.dumps({"output": output})}} + + +#: Two turns of OTel tool spans, every other one's output long enough for the default filter to compact. +SPAN_TURNS = [json.dumps([_tool_span(n) for n in range(3)]), json.dumps([_tool_span(n) for n in range(6)])] +#: The folders a judge's run stages its files under, each named afresh per run. +_RUN_FOLDERS = re.compile( + rf"({re.escape(RubricsVerifierTaskStep.TRAJECTORY_CONTAINER_DIR)}|{re.escape(always_on_agent_verifier.STAGING_DIR)})" + r"/[0-9a-f]{12}") + + +def _recorded_form(context: TaskStepContext) -> TaskStepContext: + """``context`` as a run that kept per-turn trajectory records leaves it, read back from its stored form.""" + for response in context.prompt_responses: + turns = response.target_agent_per_turn_trajectory_object_urls or [response.agent_trajectory_object_url] + response.target_agent_per_turn_trajectories = [ + {"native_format": "claude-code-stream-json", "atif": None} for _ in turns] + return TaskStepContext.from_dict(context.to_dict()) + + +#: The wall-clock day and time a chat run's user sim was shown, which its prompt step reads off this machine's clock. +_WALL_CLOCK = re.compile(r"\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) " + r"\d{1,2}\b|\b\d{1,2}:\d{2} [AP]M\b") +#: How long a captured string may be in the golden file before it's kept as its sha256 and length. +_KEPT_CHARS = 500 + + +def _normalised(value: Any, *, wall_clock: bool = False) -> Any: + text = _RUN_FOLDERS.sub(r"\1/", json.dumps(verifier_golden.normalised(value))) + return json.loads(_WALL_CLOCK.sub("", text) if wall_clock else text) + + +def _digested(value: Any) -> Any: + """``value`` with each long string as its sha256 and length, which pin it as surely and keep the file small.""" + if isinstance(value, str) and len(value) > _KEPT_CHARS: + return {"sha256": hashlib.sha256(value.encode()).hexdigest(), "chars": len(value)} + if isinstance(value, dict): + return {key: _digested(item) for key, item in value.items()} + if isinstance(value, list): + return [_digested(item) for item in value] + return value + + +async def _pinned(case: str, judged: Callable[[Callable[[TaskStepContext], TaskStepContext]], Awaitable[Any]], *, + wall_clock: bool = False) -> None: + """``judged`` builds the case's run, hands its context through the given form and returns what its judge was + given: that is the golden file's, as the run is and with the records. ``wall_clock``: the run shows this machine's + clock, which the golden file holds as ````.""" + got = _normalised(await judged(lambda context: context), wall_clock=wall_clock) + verifier_golden.check(FILE, case, _digested(got)) + if not verifier_golden.RECORDING: + assert _normalised(await judged(_recorded_form), wall_clock=wall_clock) == got + + +def _stored(name: str, text: str) -> str: + return get_config().get_object_store().put(f"judge-inputs/{name}", text.encode(), content_type="application/json", + allow_overwrite=True) + + +def _run(*, turns: int = 1, agent_url: bool = True, first_turn_missing: bool = False, texts: list[str] = TURNS + ) -> TaskStepContext: + """A prompt step's context: its turns' native trajectories stored, the step's own url the last one's.""" + urls: list[str | None] = [_stored(f"turn-{n}.json", text) for n, text in enumerate(texts[-turns:], 1)] + if first_turn_missing: + urls = [None, *urls] + return TaskStepContext(instance_id="inst-1", prompt_responses=[PromptResponse( + prompt_id="p1", response="The table for four is booked.", prompt_text="Book a table for four.", + agent_trajectory_object_url=urls[-1] if agent_url else None, + target_agent_per_turn_trajectory_object_urls=urls, + source_agent_per_turn_prompt_parts=[None, *[[{"kind": "text", "text": "And for Friday?"}]] * (len(urls) - 1)])]) + + +class _Judge: + """Every judge call answered passing, and what each was given.""" + + def __init__(self, answer: Callable[[list[str]], str] = G._binary) -> None: + self.answer = answer + self.calls: list[dict] = [] + self.files: dict[str, Any] = {} + + async def model(self, eval_prompt, *, model, context, image_blocks=None, system=None, cached_prefix=None): + self.calls.append({"model": model, "prompt": eval_prompt, "images": image_blocks, "system": system, + "cached_prefix": cached_prefix}) + return {"response": self.answer(G._ids(eval_prompt)), "trajectory_object_url": None} + + async def agent(self, *, eval_prompt, judge_a2a_url, judge_agent_card, judge_agent): + self.calls.append({"prompt": eval_prompt}) + return {"response": self.answer(G._ids(eval_prompt)), "trajectory_object_url": None} + + async def write_file_from_object(self, object_url, destination_path, **kwargs): + self.files[destination_path] = {"text": get_config().get_object_store().get(object_url).decode(), **kwargs} + + async def write_file_from_text(self, content, destination_path, **kwargs): + self.files[destination_path] = {"text": content, **kwargs} + + def given(self) -> dict: + return {"calls": self.calls, "files": dict(sorted(self.files.items()))} + + +def _model_judged(step_kw: dict, run: Callable[[], TaskStepContext], answer=G._binary): + async def judged(form): + judge = _Judge(answer) + step = G._step(**step_kw) + step._prompt_llm_judge = judge.model + await step.execute(form(run())) + return judge.given() + + return judged + + +def _agent_judged(monkeypatch, step_kw: dict, run: Callable[[], TaskStepContext]): + async def judged(form): + judge = _Judge() + + class Provider: + async def get_sandbox(self, sandbox_id): + assert sandbox_id == "sb-judge" + return judge + + monkeypatch.setattr(sandbox_provider, "get_agent_sandbox_provider", lambda: Provider()) + step = G._step(use_agent_judge=True, agent_name="judge", **step_kw) + step._invoke_judge_a2a = judge.agent + + async def configured(**kwargs): + return None + + step._configure_judge_a2a = configured + context = run() + context.deployed_agents.append(DeployedAgent(agent_name="judge", api_url="http://judge.test", + a2a_url="http://judge.test", sandbox_id="sb-judge")) + await step.execute(form(context)) + return judge.given() + + return judged + + +_SCREENSHOTS = {"use_trajectory": True, "criteria": G.EVIDENCE, "output_format": "rubric_evidence", + "trajectory_filter": G._evidence_step("rubric_evidence").trajectory_filter} +MODEL_CASES = { + "model_without_the_trajectory": ({}, _run), + "model_one_turn": ({"use_trajectory": True}, _run), + "model_one_turn_from_the_per_turn_urls": ({"use_trajectory": True}, + lambda: _run(agent_url=False, first_turn_missing=True)), + "model_one_turn_compacted": ({"use_trajectory": True, "trajectory_filter": TrajectoryFilter()}, + lambda: _run(texts=SPAN_TURNS)), + "model_two_turns": ({"use_trajectory": True}, lambda: _run(turns=2)), + "model_two_turns_compacted": ({"use_trajectory": True, "trajectory_filter": TrajectoryFilter()}, + lambda: _run(turns=2, texts=SPAN_TURNS)), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", MODEL_CASES) +async def test_a_model_judges_input_is_pinned(case): + step_kw, run = MODEL_CASES[case] + await _pinned(case, _model_judged(step_kw, run)) + + +@pytest.mark.asyncio +async def test_a_model_judges_input_graded_up_to_a_turn_is_pinned(): + def run(): + context = _run(turns=2) + context.metadata["user_overrides"] = {"step_params": {"verify": {"grade_up_to_turn": 1}}} + return context + + await _pinned("model_graded_up_to_turn_one", _model_judged({"use_trajectory": True}, run)) + + +@pytest.mark.asyncio +async def test_a_model_judges_screenshots_and_action_log_are_pinned(): + def run(): + context = _run() + context.prompt_responses[0].agent_trajectory_object_url = _stored("shots.json", G.SHOTS) + context.prompt_responses[0].target_agent_per_turn_trajectory_object_urls = [ + context.prompt_responses[0].agent_trajectory_object_url] + return context + + await _pinned("model_screenshots_per_criterion", _model_judged( + _SCREENSHOTS, run, answer=lambda ids: G._evidence(ids))) + + +@pytest.mark.asyncio +async def test_a_model_judges_run_context_for_a_prompt_another_follows_is_pinned(): + """The earlier of two prompts: what it's shown of the run is cut where the next one began, by position.""" + first_id, first = R._stored_prompt("prompt-1", "Book lunch with Dana.", "Booked for noon.") + second_id, second = R._stored_prompt("prompt-2", "Now book dinner.", "Booked for seven.") + events = [{"seq": 1, "ts": R._between(first[0]["ts"], first[1]["ts"]), "kind": "fired", "trigger_id": "lunch"}, + {"seq": 2, "ts": R._between(second[0]["ts"], second[1]["ts"]), "kind": "fired", "trigger_id": "dinner"}] + log = _stored(f"env_trigger_state/{second_id}.json", + json.dumps({"env_id": "calendar", "state": {"triggers": [], "events": events}})) + + def run(): + context = TaskStepContext(metadata={ + "a2a_conversations": {"prompt-1": first_id, "prompt-2": second_id}, + "env_trigger_state": {"calendar": {"object_url": log, "capture_step_id": "prompt-2"}}}) + for prompt_id, step_id, text, reply in (("p1", "prompt-1", "Book lunch with Dana.", "Booked for noon."), + ("p2", "prompt-2", "Now book dinner.", "Booked for seven.")): + context.prompt_responses.append(PromptResponse(prompt_id=prompt_id, response=reply, prompt_text=text, + step_id=step_id)) + return context + + await _pinned("model_run_context_of_an_earlier_prompt", _model_judged( + {"run_context_for_judge": ["conversation", "trigger_log"]}, run)) + + +AGENT_CASES = { + "agent_one_turn": ({"use_trajectory": True}, _run), + "agent_one_turn_compacted": ({"use_trajectory": True, "trajectory_filter": TrajectoryFilter()}, + lambda: _run(texts=SPAN_TURNS)), + "agent_two_turns": ({"use_trajectory": True}, lambda: _run(turns=2)), + "agent_two_turns_compacted": ({"use_trajectory": True, "trajectory_filter": TrajectoryFilter()}, + lambda: _run(turns=2, texts=SPAN_TURNS)), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", AGENT_CASES) +async def test_an_agent_judges_prompt_and_staged_files_are_pinned(monkeypatch, case): + step_kw, run = AGENT_CASES[case] + await _pinned(case, _agent_judged(monkeypatch, step_kw, run)) + + +@pytest.mark.asyncio +async def test_an_unstaged_always_on_judges_input_is_pinned(monkeypatch, tmp_path): + rig, run = await A._run(monkeypatch, tmp_path) + + async def judged(form): + verifier = A._verifier(default_model="judge-model") + captured = A._capture(monkeypatch, verifier) + await verifier.execute(form(copy.deepcopy(run))) + return captured + + await _pinned("always_on_unstaged", judged) + + +@pytest.mark.asyncio +async def test_a_staged_always_on_judges_prompt_and_files_are_pinned(monkeypatch): + run, sandbox, sources = A._staged_run(monkeypatch) + + async def judged(form): + sandbox.files.clear() + verifier = A._verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge") + captured = A._agent_judge(monkeypatch, verifier, sources) + await verifier.execute(form(copy.deepcopy(run))) + return {**captured, "files": dict(sorted(sandbox.files.items()))} + + await _pinned("always_on_staged", judged) + + +def _user_sim_calls(fake: T.FakeModel) -> list[dict]: + """What each of the user-sim judge's checks was sent, in an order its concurrency can't change.""" + return sorted(({k: v for k, v in call.items() if k != "alongside"} for call in fake.calls), + key=lambda call: json.dumps(call, sort_keys=True)) + + +@pytest.mark.asyncio +async def test_a_user_sim_judges_input_on_an_always_on_run_is_pinned(monkeypatch, tmp_path): + rig, run, prompt = await T._always_on(monkeypatch, tmp_path) + + async def judged(form): + fake = T.FakeModel() + await T._step(fake, monkeypatch, run_step_id="week").execute(form(copy.deepcopy(run))) + return _user_sim_calls(fake) + + await _pinned("user_sim", judged) + + +@pytest.mark.asyncio +async def test_a_user_sim_judges_input_on_a_chat_run_is_pinned(monkeypatch): + """A prompt step's user sim, judged from each read it stored (``metadata["user_sim_reads"]``).""" + run = await T._chat(monkeypatch) + + async def judged(form): + fake = T.FakeModel() + await T._step(fake, monkeypatch).execute(form(copy.deepcopy(run))) + return _user_sim_calls(fake) + + await _pinned("user_sim_chat", judged, wall_clock=True) + + +@pytest.mark.asyncio +async def test_what_a_scripted_verify_is_called_with_is_pinned(monkeypatch): + script = (b"import dataclasses, json\n" + b"async def verify(url, trajectory, run):\n" + b" run = json.loads(json.dumps(dataclasses.asdict(run), default=str))\n" + b" return [{'id': 'c', 'score': 1.0, 'given': {'url': url, 'trajectory': trajectory, 'run': run}}]\n") + monkeypatch.setattr(FileArtifact, "get", staticmethod(lambda id, version=None: SimpleNamespace(load=lambda: script))) + + async def judged(form): + context = _run(turns=2, first_turn_missing=True) + context.deployed_envs.append(DeployedEnv(env_id="e", env_version=1, mcp_url="http://mcp")) + step = EnvOutcomeVerifierTaskStep(id="outcome", version=None, env_id="e", file_artifact_id="s", verifier_id="v") + done = await step.execute(form(context)) + return done.metadata["verifications"]["v"]["results"][0]["given"] + + await _pinned("scripted_verify", judged) diff --git a/tst/unit/task_step/verifiers/test_judge_requests_golden.py b/tst/unit/task_step/verifiers/test_judge_requests_golden.py new file mode 100644 index 000000000..8b19120bf --- /dev/null +++ b/tst/unit/task_step/verifiers/test_judge_requests_golden.py @@ -0,0 +1,73 @@ +"""The direct judge sends LiteLLM the requests its golden file holds, first recorded on main before its call moved onto +``config.model_call``: the same model name, URL, key, params, headers and message shapes, keys in the same order, and +the same refusal of a URL with no key of its own. Re-record as ``tst/util/verifier_golden.py`` says.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import litellm +import pytest + +from agent_env.config.errors import ConfigError +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.util import verifier_golden +from tst.util.config import install_model_config + +FILE = "judge_requests.json" +ENDPOINT = "https://gw.test/v1" +_IMAGE = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} +_REFUSAL = "response_format json_schema is not supported with image inputs" + +CASES = { + "gpt_through_the_endpoint": dict(model="gpt-4.1-mini"), + "claude_through_the_endpoint": dict(model="claude-sonnet-4-6"), + "anthropic_prefixed_through_a_base_without_v1": dict(model="anthropic/claude-sonnet-4-6", base_url="https://gw.test"), + "gemini_through_the_endpoint_as_openai": dict(model="gemini-2.5-flash"), + "a_name_litellm_cant_place": dict(model="qwen3:8b"), + "a_prefixed_name_with_no_endpoint": dict(model="openai/gpt-4o", base_url=None), + "params_and_headers": dict(model="claude-sonnet-4-6", params={"custom_llm_provider": "openai", "temperature": 0}, + headers={"X-Team": "evals"}), + "the_judges_own_url_and_key": dict(model="gpt-4.1-mini", headers={"X-Team": "evals"}, overrides={ + "judge_litellm_base_url": "https://judge.test/v1", "judge_litellm_api_key": "secret:judge-key"}), + "the_runs_url_on_the_endpoint": dict(model="gpt-4.1-mini", headers={"X-Team": "evals"}, + overrides={"litellm_base_url": "https://GW.test/v1/"}), + "the_steps_url_elsewhere_with_the_runs_key": dict(model="gpt-4.1-mini", step={"default_model_api_base": + "https://step.test/v1"}, overrides={"litellm_api_key": "run-key"}), + "a_url_elsewhere_without_a_key": dict(model="gpt-4.1-mini", overrides={"judge_litellm_base_url": "https://x.test"}), + "screenshots": dict(model="claude-sonnet-4-6", call={"image_blocks": [_IMAGE]}), + "screenshots_refused_with_the_schema": dict(model="gpt-4.1-mini", call={"image_blocks": [_IMAGE]}, refuse=True), + "a_system_message_and_a_cached_run": dict(model="claude-sonnet-4-6", call={ + "system": "You check the run.", "cached_prefix": "THE RUN"}), + "a_cached_run_on_a_model_without_a_prompt_cache": dict(model="gpt-4.1-mini", call={"cached_prefix": "THE RUN"}), + "a_timeout": dict(model="gpt-4.1-mini", step={"judge_timeout_seconds": 42}), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", CASES) +async def test_the_direct_judge_sends_the_requests_it_always_has(monkeypatch, case): + spec = CASES[case] + install_model_config(monkeypatch, secrets={"judge-key": "judge-secret"}, base_url=spec.get("base_url", ENDPOINT), + api_key="configured-key", params=spec.get("params", {}), headers=spec.get("headers", {})) + requests: list[dict] = [] + + async def acompletion(**kwargs): + requests.append(kwargs) + if spec.get("refuse") and len(requests) == 1: + raise Exception(_REFUSAL) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content='{"results": []}'))]) + + monkeypatch.setattr(litellm, "acompletion", acompletion) + judge = RubricsVerifierTaskStep(id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False, + verifier_id="vt", **spec.get("step", {})) + context = TaskStepContext(metadata={"user_overrides": spec.get("overrides", {})}) + raised = None + try: + await judge._prompt_llm_judge("THE RUN then the check", model=spec["model"], context=context, + **spec.get("call", {})) + except ConfigError as error: + raised = str(error) + + verifier_golden.check(FILE, case, {"requests": requests, "raised": raised}) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py index fdc486cf9..ebd49ef74 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_base_url.py @@ -13,10 +13,9 @@ import pytest from pytest_socket import enable_socket -import agent_env.config as cfg_mod -from agent_env.config.model import ModelConfig from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, _judge_route +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.util.config import install_model_config _VERDICT = json.dumps({"results": [{"criterion_id": "c1", "passed": True}]}) @@ -67,11 +66,7 @@ def endpoint(): def _judge(monkeypatch, base_url: str, params: dict | None = None) -> RubricsVerifierTaskStep: - config = cfg_mod.Config() - monkeypatch.setattr(config, "_model_cfg", ModelConfig(base_url=base_url, api_key="k", params=params or {})) - monkeypatch.setattr(cfg_mod, "get_config", lambda: config) - monkeypatch.delenv("LITELLM_BASE_URL", raising=False) - monkeypatch.delenv("LITELLM_API_KEY", raising=False) + install_model_config(monkeypatch, base_url=base_url, api_key="k", params=params or {}) return RubricsVerifierTaskStep(id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False, verifier_id="vt") @@ -113,32 +108,3 @@ async def test_a_judge_whose_params_name_its_provider_keeps_that_providers_url(m await judge._prompt_llm_judge("grade it", model="claude-sonnet-4-6", context=TaskStepContext()) assert _Endpoint.calls == [("/v1/chat/completions", "claude-sonnet-4-6")] - - -def _provider(name: str, custom_llm_provider: str | None = None): - providers = {"claude-sonnet-4-6": "anthropic", "gpt-4.1-mini": "openai", "gemini-2.5-flash": "vertex_ai", - "deepseek/deepseek-chat": "deepseek"} - if custom_llm_provider: - return name, custom_llm_provider, None, None - if name not in providers: - print("Provider List") - raise ValueError(name) - return name, providers[name], None, None - - -def test_the_route_keeps_a_native_name_and_sends_an_unknown_one_as_openai(capsys): - assert _judge_route("claude-sonnet-4-6", "https://gw.example/v1", _provider) == ( - "claude-sonnet-4-6", "https://gw.example") - assert _judge_route("gpt-4.1-mini", "https://gw.example/v1", _provider) == ("gpt-4.1-mini", "https://gw.example/v1") - assert _judge_route("qwen3:8b", "https://gw.example", _provider) == ("openai/qwen3:8b", "https://gw.example") - assert _judge_route("qwen3:8b", None, _provider) == ("qwen3:8b", None) - assert _judge_route("claude-sonnet-4-6", "https://gw.example/v1", _provider, "openai") == ( - "claude-sonnet-4-6", "https://gw.example/v1") - assert _judge_route("qwen3:8b", "https://gw.example", _provider, "hosted_vllm") == ("qwen3:8b", "https://gw.example") - assert _judge_route("gemini-2.5-flash", "https://gw.example/v1", _provider) == ( - "openai/gemini-2.5-flash", "https://gw.example/v1") - assert _judge_route("deepseek/deepseek-chat", "https://gw.example/v1", _provider) == ( - "deepseek/deepseek-chat", "https://gw.example/v1") - assert _judge_route("gemini-2.5-flash", "https://gw.example", _provider, "vertex_ai") == ( - "gemini-2.5-flash", "https://gw.example") - assert capsys.readouterr().out == "" diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py index 56dc56d6f..afa8d9d8f 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py @@ -1,6 +1,6 @@ """The rubrics judge's calls go through the installed litellm, not a stand-in. -``_prompt_llm_judge`` is the only caller of litellm: ``acompletion`` with a strict ``json_schema`` response_format over +``_prompt_llm_judge`` calls litellm through ``config.model_call``: ``acompletion`` with a strict ``json_schema`` response_format over text or text-and-image content, read through ``.choices[0].message.content``, sent again without the response_format when an endpoint refuses it with images. Each test hands the real ``litellm.acompletion`` exactly the kwargs the judge builds, adding only litellm's ``mock_response`` so nothing leaves the process, so a litellm release that stops @@ -13,10 +13,9 @@ import litellm import pytest -from agent_env.config.model import ModelConfig -from agent_env.store import LocalSecretStore from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.util.config import install_model_config _VERDICT = {"results": [{"criterion_id": "c1", "passed": True}]} _IMAGE = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} @@ -24,14 +23,7 @@ @pytest.fixture def judge(monkeypatch): - import agent_env.config as cfg_mod - - config = cfg_mod.Config() - config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) - monkeypatch.setattr(config, "_model_cfg", ModelConfig(base_url="https://judge.invalid/v1")) - monkeypatch.setattr(cfg_mod, "get_config", lambda: config) - monkeypatch.delenv("LITELLM_BASE_URL", raising=False) - monkeypatch.delenv("LITELLM_API_KEY", raising=False) + install_model_config(monkeypatch, secrets={"litellm_api_key": "k"}, base_url="https://judge.invalid/v1") return RubricsVerifierTaskStep( id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False, default_model_api_base="https://judge.invalid/v1", verifier_id="vt", @@ -94,7 +86,8 @@ async def test_a_default_call_sends_one_user_message_and_returns_its_usage_and_c assert calls[0]["messages"] == [{"role": "user", "content": "grade it"}] assert out["usage"]["total_tokens"] > 0 - assert isinstance(out["cost"], float) and out["cost"] > 0 + assert (out["model"], out["endpoint_host"], out["cost_status"]) == ("openai/gpt-4o", "judge.invalid", "priced") + assert out["cost_source"].startswith("litellm@") and out["cost_usd"] > 0 and out["reported_cost_usd"] is None @pytest.mark.asyncio diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py index 0ec935838..24723b2c1 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_screenshots.py @@ -12,7 +12,6 @@ import pytest from agent_env.config.errors import ConfigError -from agent_env.config.model import ModelConfig from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import SCREENSHOT_GROUNDING from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( @@ -30,6 +29,7 @@ final_frames_from_raw, strip_base64_images, ) +from tst.util.config import install_model_config def _expected_block(b64: str) -> dict: @@ -401,16 +401,8 @@ async def test_prompt_llm_judge_base_url_precedence(monkeypatch): import types import litellm - import agent_env.config as cfg_mod - from agent_env.store import LocalSecretStore - - real_cfg = cfg_mod.Config() - real_cfg.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) - monkeypatch.setattr(real_cfg, "_model_cfg", ModelConfig()) - monkeypatch.setattr(cfg_mod, "get_config", lambda: real_cfg) - monkeypatch.delenv("LITELLM_BASE_URL", raising=False) - monkeypatch.delenv("LITELLM_API_KEY", raising=False) + install_model_config(monkeypatch, secrets={"litellm_api_key": "k"}) calls: list[dict] = [] diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 7b7d86e56..dd38dd006 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -15,7 +15,6 @@ from agent_env.config.model import ModelConfig, resolve_model from agent_env.openenv.reward import reward_of -from agent_env.store import LocalSecretStore from agent_env.task import Task from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError from agent_env.task_step.registry import _builtin_registry @@ -44,6 +43,7 @@ from tst.unit.task_step.test_prompt_agent_built_in_user_sim import _step as _chat_step from tst.unit.task_step.test_prompt_agent_built_in_user_sim import said from tst.unit.task_step.test_prompt_agent_user_side import AGENT, USER, Peers, text +from tst.util.config import install_model_config JUDGE = "fake/judge" OTHER_PERSONA = "0" * 64 # a persona fingerprint no run here has @@ -106,7 +106,7 @@ async def __call__(self, eval_prompt, *, model, context, image_blocks=None, syst @staticmethod def _reply(text: str) -> dict: return {"response": text, "trajectory_object_url": None, - "usage": {"prompt_tokens": 100, "completion_tokens": 20}, "cost": 0.001} + "usage": {"prompt_tokens": 100, "completion_tokens": 20}, "cost_usd": 0.001} def asked(self, name: str) -> list[dict]: return [c for c in self.calls if name in c["names"]] @@ -279,6 +279,29 @@ async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(monkey assert entry["usage"]["prompt_tokens"] == 700 and entry["cost"] == pytest.approx(0.007) +class _PricedFirst(FakeModel): + """The fake, its first ``priced`` answers costing $0.001 each and the rest without a figure.""" + + def __init__(self, priced: int): + super().__init__() + self.priced, self.answered = priced, 0 + + async def __call__(self, *args, **kwargs): + got = await super().__call__(*args, **kwargs) + self.answered += 1 + return {**got, "cost_usd": 0.001 if self.answered <= self.priced else None} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("priced, cost, unpriced", [(7, 0.007, None), (3, 0.003, 4), (0, None, 7)]) +async def test_the_runs_cost_sums_the_calls_with_a_figure_and_counts_those_without(monkeypatch, priced, cost, + unpriced): + entry = await _judge(_run(("reply", "ok")), _PricedFirst(priced), monkeypatch) + + assert entry["calls"] == 7 and entry.get("unpriced_calls") == unpriced + assert entry["cost"] is None if cost is None else entry["cost"] == pytest.approx(cost) + + @pytest.mark.asyncio async def test_each_call_shares_the_run_and_asks_one_check_and_the_first_call_runs_alone(monkeypatch): fake = FakeModel() @@ -696,14 +719,7 @@ def test_the_run_is_the_one_run_step_id_names(): # ---- the judge's calls, through litellm ----------------------------------------------------------------------------- @pytest.fixture def litellm_config(monkeypatch): - import agent_env.config as cfg_mod - - config = cfg_mod.Config() - config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) - monkeypatch.setattr(config, "_model_cfg", ModelConfig()) - monkeypatch.setattr(cfg_mod, "get_config", lambda: config) - monkeypatch.delenv("LITELLM_BASE_URL", raising=False) - monkeypatch.delenv("LITELLM_API_KEY", raising=False) + install_model_config(monkeypatch, secrets={"litellm_api_key": "k"}) def _litellm_answers(monkeypatch, answer): diff --git a/tst/unit/training/harness_test.py b/tst/unit/training/harness_test.py index 99487146e..347089dea 100644 --- a/tst/unit/training/harness_test.py +++ b/tst/unit/training/harness_test.py @@ -123,6 +123,35 @@ def bundle(tmp_path, monkeypatch, local_stores): # noqa: F811 "tasks/self-judged.json": json.dumps([DEPLOY, ASK, JUDGE])}) +def test_a_task_left_out_isnt_linted_so_a_config_only_it_gets_wrong_cant_stop_the_rest(bundle, caplog): + unprobed = {"id": "files", "type": "verify_sandbox", "sandbox_name": "box", + "criteria": [{"type": "probe_file_exists", "criterion": "has it"}]} + (bundle / "tasks/sandboxed.json").write_text(json.dumps([DEPLOY, unprobed])) + + with caplog.at_level(logging.WARNING, logger=harness.__name__): + rows = prompt_rows(bundle) + + assert "sandboxed is left out" in caplog.text + assert not any(row["task_id"].endswith("sandboxed") for row in rows) + + +def test_the_dataset_and_the_sessions_prepare_their_tasks_with_skip_lint_as_given(bundle, monkeypatch): + prepared = [] + + def prepare(*args, **kwargs): + prepared.append(kwargs["skip_lint"]) + raise RuntimeError("prepared") + + monkeypatch.setattr(harness, "prepare_bundle", prepare) + for skip in (False, True): + with pytest.raises(RuntimeError, match="prepared"): + prompt_rows(bundle, skip_lint=skip) + with pytest.raises(RuntimeError, match="prepared"): + AgentEnvSessionFactory(bundle, llm_url="http://127.0.0.1:1", model=MODEL, skip_lint=skip) + + assert prepared == [False, False, True, True] + + def test_the_solver_alone_calls_the_policy_through_the_proxy_and_the_run_is_captured_and_scored(engine, bundle, caplog): with caplog.at_level(logging.WARNING, logger=harness.__name__): rows = prompt_rows(bundle) diff --git a/tst/util/config.py b/tst/util/config.py index b7135a02b..303f86700 100644 --- a/tst/util/config.py +++ b/tst/util/config.py @@ -3,7 +3,8 @@ `config_with_document` lives here rather than in `agent_env.config`: nothing in the shipped package needs a way to choose a document without writing a file, so shipping one would be production surface with no production caller. A test is allowed to know internals the -package should not expose. +package should not expose. `install_model_config` is the same: a test's `[model]` installed +as the process-wide config, which the package never needs to do. """ from __future__ import annotations @@ -11,7 +12,11 @@ from pathlib import Path from typing import Any, Mapping, Optional +import pytest + from agent_env.config import runtime, snapshot +from agent_env.config.model import ENV_API_KEY, ENV_BASE_URL, ModelConfig +from agent_env.store import LocalSecretStore def config_with_document(document: Mapping[str, Any], *, path: Optional[Path] = None, @@ -24,3 +29,18 @@ def config_with_document(document: Mapping[str, Any], *, path: Optional[Path] = config = runtime.Config(**kwargs) config._snapshot = snapshot.Snapshot(path=path, _document=document) return config + + +def install_model_config(monkeypatch: pytest.MonkeyPatch, *, secrets: Optional[Mapping[str, str]] = None, + **model: Any) -> runtime.Config: + """The process-wide `Config` for this test, its `[model]` `ModelConfig(**model)` and its secrets `secrets` (the + environment's otherwise), with `LITELLM_BASE_URL` and `LITELLM_API_KEY` cleared. Installed as the singleton, so + every `get_config()` returns it, wherever it was imported from.""" + config = runtime.Config() + if secrets is not None: + config.set_secret_store(LocalSecretStore(values=dict(secrets), use_env=False)) + monkeypatch.setattr(config, "_model_cfg", ModelConfig(**model)) + monkeypatch.setattr(runtime, "_config", config) + monkeypatch.delenv(ENV_BASE_URL, raising=False) + monkeypatch.delenv(ENV_API_KEY, raising=False) + return config diff --git a/tst/util/verifier_golden.py b/tst/util/verifier_golden.py index 357f0095d..62fd910f2 100644 --- a/tst/util/verifier_golden.py +++ b/tst/util/verifier_golden.py @@ -1,7 +1,8 @@ -"""Golden files for the built-in verifiers' legacy ``verifications`` entries, under ``tst/data/verifier_golden``: first -recorded on main before the verifiers moved onto ``VerifierTaskStep``, and the contract since. A run must leave what the -file holds, number types and key order included. After a deliberate change, re-record with -``AGENTENV_TEST_RECORD_GOLDEN=1`` and review the files' diff: each check then writes what the run left instead.""" +"""Golden files under ``tst/data/verifier_golden``: the built-in verifiers' legacy ``verifications`` entries, first +recorded on main before the verifiers moved onto ``VerifierTaskStep``, and the requests the direct judge hands LiteLLM +(``judge_requests.json``), first recorded on main before its call moved onto ``config.model_call``; each the contract +since. A run must leave what the file holds, number types and key order included. After a deliberate change, re-record +with ``AGENTENV_TEST_RECORD_GOLDEN=1`` and review the files' diff: each check then writes what the run left instead.""" from __future__ import annotations diff --git a/uv.lock b/uv.lock index 08719bcea..fd91eafec 100644 --- a/uv.lock +++ b/uv.lock @@ -48,7 +48,7 @@ http-server = [ [[package]] name = "agentenv-framework" -version = "0.9.1603" +version = "0.9.1609" source = { editable = "." } dependencies = [ { name = "a2a-sdk" }, @@ -289,7 +289,7 @@ provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "mi [[package]] name = "agentenv-framework-protocol" -version = "0.1.618" +version = "0.1.624" source = { editable = "packages/agentenv-protocol" } dependencies = [ { name = "httpx" },