Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/agent_env/agents/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn.
`provider_details.unreported_calls`. Report the turn's usage, never the session's: a CLI that resumes a session can
report the session's totals, as Claude Code's `modelUsage` and `total_cost_usd` do, so count the turn as the
difference from the session's totals before it, and check that it covers every call the turn made, subagents'
included (Claude Code's `result` events count only the main agent's). Code that sets these fields needs a
included (Claude Code's `result` events count only the main agent's). A cost the runtime couldn't price is unset,
never 0. Code that sets these fields needs a
`requirements.txt` protocol floor at the first release that has them.
- **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't
happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text
Expand Down
38 changes: 21 additions & 17 deletions src/agent_env/agents/agents/browser-use/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,10 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult:
error = (await stderr).decode(errors="replace")[-4000:] or f"agent.py exited {process.returncode}"
return TaskResult.failure("browser_use.crashed", error, error_type="infra_error")
tool_calls = 0
for message in result["messages"]:
for index, message in enumerate(result["messages"]):
request.trajectory.append(message)
tool_calls += sum(part.get("part_kind") == "tool-call" for part in message.get("parts") or [])
usage = result["usage"]
if index >= result.get("prior", 0): # the context's earlier turns counted theirs
tool_calls += sum(part.get("part_kind") == "tool-call" for part in message.get("parts") or [])
builder = TaskResult.builder()
if result["status"] == "completed":
builder.succeeded()
Expand All @@ -156,24 +156,28 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult:
return (
builder.add_text(result["text"] or result.get("error") or result["status"])
.session_ref(session)
.usage(Usage(
tool_call_count=tool_calls,
input_tokens=usage["input"],
output_tokens=usage["output"],
total_tokens=usage["input"] + usage["output"],
# browser-use counts the cache in its input on both of its routes: Anthropic's Messages, whose cache
# it adds in, and OpenAI's Chat Completions, whose prompt already holds it.
cache_read_tokens=usage["cache_read"],
cache_write_tokens=usage["cache_write"],
input_includes_cache=True,
cost_usd=result.get("cost_usd"),
provider_details={"steps": result["steps"], "model": result["model"], "requests": usage["requests"],
"success": result.get("success")},
))
.usage(_usage(result, tool_calls))
.build()
)


def _usage(result: dict, tool_calls: int) -> Usage:
"""The turn's usage as browser-use counts it, over the responses it got. browser-use counts the cache in its input
on both of its routes: Anthropic's Messages, whose cache it adds in, and OpenAI's Chat Completions, whose prompt
already holds it. A request sent and not refused that got no response it counts, cut off by the run's timeout or
retried, is counted in ``unreported_calls``; when no request got one, the counts are unknown, not zero."""
usage, calls = result["usage"], result.get("calls") or {}
unreported = max(calls.get("sent", 0) - calls.get("refused", 0) - usage["requests"], 0)
details = {"steps": result["steps"], "model": result["model"], "requests": usage["requests"],
"success": result.get("success"), **({"unreported_calls": unreported} if unreported else {})}
if unreported and not usage["requests"]:
return Usage(tool_call_count=tool_calls, provider_details=details)
return Usage(tool_call_count=tool_calls, input_tokens=usage["input"], output_tokens=usage["output"],
total_tokens=usage["input"] + usage["output"], cache_read_tokens=usage["cache_read"],
cache_write_tokens=usage["cache_write"], input_includes_cache=True, cost_usd=result.get("cost_usd"),
provider_details=details)


agent = BrowserUseAgent()
app = agent.create_app()
AGENT_CARD = app.state.agentenv_a2a.card
Expand Down
30 changes: 25 additions & 5 deletions src/agent_env/agents/agents/browser-use/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@
from urllib.parse import urlsplit, urlunsplit

import httpx2
from anthropic import AsyncAnthropic
from anthropic import AsyncAnthropic, DefaultAsyncHttpxClient as AnthropicHttpxClient
from browser_use import Agent, Browser, Tools
from browser_use.browser.browser import _daemon_request
from browser_use.tools import TOOL_RETRIES
from mcp import ClientSession
from mcp.client.streamable_http import streamable_http_client
from openai import AsyncOpenAI
from openai import AsyncOpenAI, DefaultAsyncHttpxClient as OpenAIHttpxClient
from pydantic import TypeAdapter
from pydantic_ai import ModelRetry, Tool
from pydantic_ai.messages import ModelMessagesTypeAdapter, ModelRequest
Expand All @@ -37,6 +37,21 @@

TOOL_NAME = re.compile(r"[^a-zA-Z0-9_-]")
_CHECKPOINT = TypeAdapter(ModelRequest | None)
# The model requests the turn sent, and those the endpoint refused (an error answer bills nothing): a request cut off
# before its response, by the run's timeout or a retry, has no usage in the run's own count.
CALLS = {"sent": 0, "refused": 0}


async def _sent(request) -> None:
CALLS["sent"] += 1


async def _answered(response) -> None:
if response.status_code >= 400:
CALLS["refused"] += 1


HOOKS = {"request": [_sent], "response": [_answered]}


def _api_root(url: str) -> str:
Expand All @@ -52,10 +67,12 @@ def model(turn: dict):
name, base_url, key = turn["model"], os.environ["LITELLM_BASE_URL"], os.environ["LITELLM_API_KEY"]
headers = json.loads(os.environ.get("LITELLM_EXTRA_HEADERS") or "{}")
if "claude" in name.lower():
client = AsyncAnthropic(auth_token=key, base_url=_api_root(base_url), default_headers=headers)
client = AsyncAnthropic(auth_token=key, base_url=_api_root(base_url), default_headers=headers,
http_client=AnthropicHttpxClient(event_hooks=HOOKS))
return AnthropicModel(name, provider=AnthropicProvider(anthropic_client=client),
profile=anthropic_model_profile(name.rsplit("/", 1)[-1]))
client = AsyncOpenAI(api_key=key, base_url=base_url, default_headers=headers)
client = AsyncOpenAI(api_key=key, base_url=base_url, default_headers=headers,
http_client=OpenAIHttpxClient(event_hooks=HOOKS))
settings = {"openai_reasoning_effort": turn["effort"]} if turn["effort"] else None
return OpenAIChatModel(name, provider=OpenAIProvider(openai_client=client), settings=settings)

Expand Down Expand Up @@ -144,13 +161,16 @@ async def main() -> None:
stack.push_async_callback(agent.close)
if history.exists():
restore(agent, history)
# The run's messages are the conversation's; the turn's own start here, as browser-use slices its usage.
prior = len(agent._history or [])
result = await agent.run(**{key: turn[key] for key in ("max_steps", "timeout") if turn.get(key)})
if agent._history:
save(agent, history)
print(json.dumps({"type": "result", "result": {
"status": result.status, "text": result.text, "success": result.success, "error": result.error,
"steps": result.steps, "usage": result.usage, "cost_usd": result.cost_usd, "model": turn["model"],
"messages": ModelMessagesTypeAdapter.dump_python(result.messages, mode="json"),
"messages": ModelMessagesTypeAdapter.dump_python(result.messages, mode="json"), "prior": prior,
"calls": CALLS,
}}), flush=True)


Expand Down
7 changes: 5 additions & 2 deletions src/agent_env/agents/agents/hermes/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ the model no one is there to answer.
| `peer-agents/v1` | SDK default: Hermes reaches the task's other agents through the SDK's `peer_agents` MCP tools |
| `attribution-probe/v1` | the `project_id` and `task_id` the last turn sent |

A turn's usage is Hermes' own report for its main model calls: tokens, `api_calls`, and its estimated cost where the
endpoint's model list prices the model. The image is `python:3.11-slim` with Hermes, `pdftoppm` and the agent, running
A turn's usage counts every model call the turn made: its main calls and its auxiliary ones (title generation,
compression), as Hermes' usage file reports them, and a `delegate_task` child's, from their `post_api_request` events,
which the file leaves out. A turn killed before it wrote the file keeps the calls its events counted, marked as a lower
bound. Hermes' estimated cost is kept only where it priced the calls. The profile turns Hermes' background review off:
its fork outlives the turn's process, which kills it before it reports what the proxy billed. The image is `python:3.11-slim` with Hermes, `pdftoppm` and the agent, running
as the `agent` user. Hermes' runtime (its own Python, Node and ripgrep) and the profiles are under `/data/.hermes`,
whatever the uid. It has no `install/v1`.
100 changes: 77 additions & 23 deletions src/agent_env/agents/agents/hermes/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
from hermes_bridge import (
ERROR_MESSAGE_LIMIT,
HERMES_ROOT,
CallCounts,
get_last_attribution_snapshot,
kill_running_clis,
profile_dir,
Expand Down Expand Up @@ -518,50 +519,103 @@ async def swap() -> None:
return {"context_id": target_context_id}


def _usage(result) -> Usage:
"""The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with
``input_tokens`` counting the cache. Hermes' ``input_tokens`` is the prompt less its cache reads and writes, and its
total counts them, so they're added back; a total of the input and output alone says the input already holds them,
and a total that says neither keeps Hermes' rule with the flag unknown. Calls that reported no counts at all got
none from their endpoint, as streamed Claude answers through a LiteLLM proxy, so the turn's counts are unknown
rather than nothing."""
report = result.usage or {}
# The cost statuses under which Hermes' figure is a price; under any other (``unknown``), its 0.0 means unpriced.
_PRICED = ("estimated", "actual")

def count(name: str) -> int | None:
value = report.get(name)
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None

cost = report.get("estimated_cost_usd")
details = {key: report[key] for key in ("api_calls", "reasoning_tokens", "cost_status", "turn_exit_reason")
def _count(report: dict, name: str) -> int | None:
value = report.get(name)
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None


def _usage(result) -> Usage:
"""The turn's tool calls and token counts, with ``input_tokens`` counting the cache: its main model calls as
Hermes' usage file counts them, its auxiliary calls (title generation, compression) as the file's ``auxiliary``
block counts them, and a delegate_task child's calls, which the file leaves out, from their events. Hermes'
``input_tokens`` is the prompt less its cache reads and writes, and its total counts them, so they're added back; a
total of the input and output alone says the input already holds them, and a total that says neither keeps
Hermes' rule with the flag unknown. Calls that reported no counts at all got none from their endpoint, as streamed
Claude answers through a LiteLLM proxy once did, so the turn's counts are unknown rather than nothing. A turn killed
before it wrote its file counts the calls it made from their events, and one more call as unreported: the one in
flight, or its title call. Each part whose calls reported no counts is counted in ``unreported_calls`` while the
others' counts are kept. Hermes' cost is kept only when it priced every call the turn made."""
if result.killed_calls is not None:
return _counted_calls(result.tool_call_count, result.killed_calls, unreported=1)
report = result.usage or {}
aux = report.get("auxiliary") if isinstance(report.get("auxiliary"), dict) else {}
side = result.side_calls
calls = (report.get("total_including_auxiliary") or {}).get("api_calls", report.get("api_calls"))
details = {key: report[key] for key in ("reasoning_tokens", "cost_status", "turn_exit_reason")
if report.get(key) is not None}
counts = {name: count(name) for name in ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens",
"cache_write_tokens")}
if (count("api_calls") or 0) > 0 and not any(counts.values()):
return Usage(tool_call_count=result.tool_call_count,
provider_details={**details, "unreported_calls": details["api_calls"]})
if calls is not None or side is not None:
details["api_calls"] = (calls or 0) + (side.calls if side else 0)
counts = {name: _count(report, name) for name in ("input_tokens", "output_tokens", "total_tokens",
"cache_read_tokens", "cache_write_tokens")}
unreported = side.unreported if side is not None else 0
if (_count(report, "api_calls") or 0) > 0 and not any(counts.values()):
unreported += _count(report, "api_calls")
counts = dict.fromkeys(counts)
# Hermes' auxiliary counts are its canonical ones: input without the cache.
extra = {"input": sum(_count(aux, key) or 0 for key in ("input_tokens", "cache_read_tokens", "cache_write_tokens")),
"output": _count(aux, "output_tokens") or 0, "read": _count(aux, "cache_read_tokens") or 0,
"write": _count(aux, "cache_write_tokens") or 0}
if (_count(aux, "api_calls") or 0) > 0 and not (extra["input"] or extra["output"]):
unreported += _count(aux, "api_calls")
if side is not None:
extra = {"input": extra["input"] + side.input_tokens, "output": extra["output"] + side.output_tokens,
"read": extra["read"] + side.cache_read_tokens, "write": extra["write"] + side.cache_write_tokens}
if unreported:
details["unreported_calls"] = unreported
read, write = counts["cache_read_tokens"], counts["cache_write_tokens"]
input_tokens, output, total = counts["input_tokens"], counts["output_tokens"] or 0, counts["total_tokens"]
input_tokens, output, total = counts["input_tokens"], counts["output_tokens"], counts["total_tokens"]
cached = (read or 0) + (write or 0)
includes_cache = None
if input_tokens is not None:
if cached == 0 or total == input_tokens + output:
if cached == 0 or total == input_tokens + (output or 0):
includes_cache = True
else:
includes_cache = True if total == input_tokens + output + cached else None
includes_cache = True if total == input_tokens + (output or 0) + cached else None
input_tokens += cached
input_tokens += extra["input"]
output = None if output is None else output + extra["output"]
total = None if total is None else total + extra["input"] + extra["output"]
read = None if read is None else read + extra["read"]
write = None if write is None else write + extra["write"]
elif extra["input"] or extra["output"]:
input_tokens, output, total = extra["input"], extra["output"], extra["input"] + extra["output"]
read, write, includes_cache = extra["read"], extra["write"], True
elif unreported:
return Usage(tool_call_count=result.tool_call_count, provider_details=details)
cost = report.get("estimated_cost_usd")
if aux:
cost = (report.get("total_including_auxiliary") or {}).get("estimated_cost_usd")
# Hermes' cost leaves out a delegate_task child's calls, so a turn with some has none.
priced = (report.get("cost_status") in _PRICED and isinstance(cost, (int, float)) and not isinstance(cost, bool)
and not unreported and not (side and side.calls))
return Usage(
tool_call_count=result.tool_call_count,
input_tokens=input_tokens,
output_tokens=counts["output_tokens"],
output_tokens=output,
total_tokens=total,
cache_read_tokens=read,
cache_write_tokens=write,
input_includes_cache=includes_cache,
cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None,
cost_usd=cost if priced and cost >= 0 else None,
provider_details=details,
)


def _counted_calls(tool_calls: int, counted: CallCounts, unreported: int = 0) -> Usage:
"""Calls counted from their events: their counts, those without usage and ``unreported`` more counted apart."""
details = {"api_calls": counted.calls, "unreported_calls": counted.unreported + unreported}
if counted.calls == counted.unreported:
return Usage(tool_call_count=tool_calls, provider_details=details)
return Usage(tool_call_count=tool_calls, input_tokens=counted.input_tokens, output_tokens=counted.output_tokens,
total_tokens=counted.input_tokens + counted.output_tokens,
cache_read_tokens=counted.cache_read_tokens, cache_write_tokens=counted.cache_write_tokens,
input_includes_cache=True, provider_details=details)


def _replace_workspace(staged: Path, then: Callable[[], None] | None = None) -> None:
"""Swap staged in for the workspace's contents. When then() raises after the swap, the previous contents come
back."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
"post_tool_call": ("session_id", "turn_id", "tool_call_id", "tool_name", "result", "status", "duration_ms",
"error_type", "error_message"),
# ``response`` is Hermes' sanitized record of the model's answer: its assistant message, tool calls and usage.
"post_api_request": ("session_id", "turn_id", "api_call_count", "model", "finish_reason", "response"),
# ``usage`` is the call's token counts, normalized (``prompt_tokens`` counts the cache), never truncated.
"post_api_request": ("session_id", "turn_id", "api_call_count", "model", "finish_reason", "response", "usage"),
"api_request_error": ("session_id", "turn_id", "api_call_count", "model", "status_code", "reason", "retryable",
"error"),
"post_llm_call": ("session_id", "turn_id", "assistant_response"),
Expand Down
Loading
Loading