diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index b0e5f6fa6..c7f687c06 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -60,7 +60,8 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn. `provider_details.unreported_calls`. Report the turn's usage, never the session's: a CLI that resumes a session can report the session's totals, as Claude Code's `modelUsage` and `total_cost_usd` do, so count the turn as the difference from the session's totals before it, and check that it covers every call the turn made, subagents' - included (Claude Code's `result` events count only the main agent's). Code that sets these fields needs a + included (Claude Code's `result` events count only the main agent's). A cost the runtime couldn't price is unset, + never 0. Code that sets these fields needs a `requirements.txt` protocol floor at the first release that has them. - **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text diff --git a/src/agent_env/agents/agents/browser-use/a2a_server.py b/src/agent_env/agents/agents/browser-use/a2a_server.py index 3146f8910..2ddb03fd9 100644 --- a/src/agent_env/agents/agents/browser-use/a2a_server.py +++ b/src/agent_env/agents/agents/browser-use/a2a_server.py @@ -144,10 +144,10 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult: error = (await stderr).decode(errors="replace")[-4000:] or f"agent.py exited {process.returncode}" return TaskResult.failure("browser_use.crashed", error, error_type="infra_error") tool_calls = 0 - for message in result["messages"]: + for index, message in enumerate(result["messages"]): request.trajectory.append(message) - tool_calls += sum(part.get("part_kind") == "tool-call" for part in message.get("parts") or []) - usage = result["usage"] + if index >= result.get("prior", 0): # the context's earlier turns counted theirs + tool_calls += sum(part.get("part_kind") == "tool-call" for part in message.get("parts") or []) builder = TaskResult.builder() if result["status"] == "completed": builder.succeeded() @@ -156,24 +156,28 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult: return ( builder.add_text(result["text"] or result.get("error") or result["status"]) .session_ref(session) - .usage(Usage( - tool_call_count=tool_calls, - input_tokens=usage["input"], - output_tokens=usage["output"], - total_tokens=usage["input"] + usage["output"], - # browser-use counts the cache in its input on both of its routes: Anthropic's Messages, whose cache - # it adds in, and OpenAI's Chat Completions, whose prompt already holds it. - cache_read_tokens=usage["cache_read"], - cache_write_tokens=usage["cache_write"], - input_includes_cache=True, - cost_usd=result.get("cost_usd"), - provider_details={"steps": result["steps"], "model": result["model"], "requests": usage["requests"], - "success": result.get("success")}, - )) + .usage(_usage(result, tool_calls)) .build() ) +def _usage(result: dict, tool_calls: int) -> Usage: + """The turn's usage as browser-use counts it, over the responses it got. browser-use counts the cache in its input + on both of its routes: Anthropic's Messages, whose cache it adds in, and OpenAI's Chat Completions, whose prompt + already holds it. A request sent and not refused that got no response it counts, cut off by the run's timeout or + retried, is counted in ``unreported_calls``; when no request got one, the counts are unknown, not zero.""" + usage, calls = result["usage"], result.get("calls") or {} + unreported = max(calls.get("sent", 0) - calls.get("refused", 0) - usage["requests"], 0) + details = {"steps": result["steps"], "model": result["model"], "requests": usage["requests"], + "success": result.get("success"), **({"unreported_calls": unreported} if unreported else {})} + if unreported and not usage["requests"]: + return Usage(tool_call_count=tool_calls, provider_details=details) + return Usage(tool_call_count=tool_calls, input_tokens=usage["input"], output_tokens=usage["output"], + total_tokens=usage["input"] + usage["output"], cache_read_tokens=usage["cache_read"], + cache_write_tokens=usage["cache_write"], input_includes_cache=True, cost_usd=result.get("cost_usd"), + provider_details=details) + + agent = BrowserUseAgent() app = agent.create_app() AGENT_CARD = app.state.agentenv_a2a.card diff --git a/src/agent_env/agents/agents/browser-use/agent.py b/src/agent_env/agents/agents/browser-use/agent.py index 324e2c3ed..65f616052 100644 --- a/src/agent_env/agents/agents/browser-use/agent.py +++ b/src/agent_env/agents/agents/browser-use/agent.py @@ -19,13 +19,13 @@ from urllib.parse import urlsplit, urlunsplit import httpx2 -from anthropic import AsyncAnthropic +from anthropic import AsyncAnthropic, DefaultAsyncHttpxClient as AnthropicHttpxClient from browser_use import Agent, Browser, Tools from browser_use.browser.browser import _daemon_request from browser_use.tools import TOOL_RETRIES from mcp import ClientSession from mcp.client.streamable_http import streamable_http_client -from openai import AsyncOpenAI +from openai import AsyncOpenAI, DefaultAsyncHttpxClient as OpenAIHttpxClient from pydantic import TypeAdapter from pydantic_ai import ModelRetry, Tool from pydantic_ai.messages import ModelMessagesTypeAdapter, ModelRequest @@ -37,6 +37,21 @@ TOOL_NAME = re.compile(r"[^a-zA-Z0-9_-]") _CHECKPOINT = TypeAdapter(ModelRequest | None) +# The model requests the turn sent, and those the endpoint refused (an error answer bills nothing): a request cut off +# before its response, by the run's timeout or a retry, has no usage in the run's own count. +CALLS = {"sent": 0, "refused": 0} + + +async def _sent(request) -> None: + CALLS["sent"] += 1 + + +async def _answered(response) -> None: + if response.status_code >= 400: + CALLS["refused"] += 1 + + +HOOKS = {"request": [_sent], "response": [_answered]} def _api_root(url: str) -> str: @@ -52,10 +67,12 @@ def model(turn: dict): name, base_url, key = turn["model"], os.environ["LITELLM_BASE_URL"], os.environ["LITELLM_API_KEY"] headers = json.loads(os.environ.get("LITELLM_EXTRA_HEADERS") or "{}") if "claude" in name.lower(): - client = AsyncAnthropic(auth_token=key, base_url=_api_root(base_url), default_headers=headers) + client = AsyncAnthropic(auth_token=key, base_url=_api_root(base_url), default_headers=headers, + http_client=AnthropicHttpxClient(event_hooks=HOOKS)) return AnthropicModel(name, provider=AnthropicProvider(anthropic_client=client), profile=anthropic_model_profile(name.rsplit("/", 1)[-1])) - client = AsyncOpenAI(api_key=key, base_url=base_url, default_headers=headers) + client = AsyncOpenAI(api_key=key, base_url=base_url, default_headers=headers, + http_client=OpenAIHttpxClient(event_hooks=HOOKS)) settings = {"openai_reasoning_effort": turn["effort"]} if turn["effort"] else None return OpenAIChatModel(name, provider=OpenAIProvider(openai_client=client), settings=settings) @@ -144,13 +161,16 @@ async def main() -> None: stack.push_async_callback(agent.close) if history.exists(): restore(agent, history) + # The run's messages are the conversation's; the turn's own start here, as browser-use slices its usage. + prior = len(agent._history or []) result = await agent.run(**{key: turn[key] for key in ("max_steps", "timeout") if turn.get(key)}) if agent._history: save(agent, history) print(json.dumps({"type": "result", "result": { "status": result.status, "text": result.text, "success": result.success, "error": result.error, "steps": result.steps, "usage": result.usage, "cost_usd": result.cost_usd, "model": turn["model"], - "messages": ModelMessagesTypeAdapter.dump_python(result.messages, mode="json"), + "messages": ModelMessagesTypeAdapter.dump_python(result.messages, mode="json"), "prior": prior, + "calls": CALLS, }}), flush=True) diff --git a/src/agent_env/agents/agents/hermes/README.md b/src/agent_env/agents/agents/hermes/README.md index ebaf5e09c..822785e0c 100644 --- a/src/agent_env/agents/agents/hermes/README.md +++ b/src/agent_env/agents/agents/hermes/README.md @@ -43,7 +43,10 @@ the model no one is there to answer. | `peer-agents/v1` | SDK default: Hermes reaches the task's other agents through the SDK's `peer_agents` MCP tools | | `attribution-probe/v1` | the `project_id` and `task_id` the last turn sent | -A turn's usage is Hermes' own report for its main model calls: tokens, `api_calls`, and its estimated cost where the -endpoint's model list prices the model. The image is `python:3.11-slim` with Hermes, `pdftoppm` and the agent, running +A turn's usage counts every model call the turn made: its main calls and its auxiliary ones (title generation, +compression), as Hermes' usage file reports them, and a `delegate_task` child's, from their `post_api_request` events, +which the file leaves out. A turn killed before it wrote the file keeps the calls its events counted, marked as a lower +bound. Hermes' estimated cost is kept only where it priced the calls. The profile turns Hermes' background review off: +its fork outlives the turn's process, which kills it before it reports what the proxy billed. The image is `python:3.11-slim` with Hermes, `pdftoppm` and the agent, running as the `agent` user. Hermes' runtime (its own Python, Node and ripgrep) and the profiles are under `/data/.hermes`, whatever the uid. It has no `install/v1`. diff --git a/src/agent_env/agents/agents/hermes/a2a_server.py b/src/agent_env/agents/agents/hermes/a2a_server.py index e250f708d..f8b2d4307 100644 --- a/src/agent_env/agents/agents/hermes/a2a_server.py +++ b/src/agent_env/agents/agents/hermes/a2a_server.py @@ -65,6 +65,7 @@ from hermes_bridge import ( ERROR_MESSAGE_LIMIT, HERMES_ROOT, + CallCounts, get_last_attribution_snapshot, kill_running_clis, profile_dir, @@ -518,50 +519,103 @@ async def swap() -> None: return {"context_id": target_context_id} -def _usage(result) -> Usage: - """The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with - ``input_tokens`` counting the cache. Hermes' ``input_tokens`` is the prompt less its cache reads and writes, and its - total counts them, so they're added back; a total of the input and output alone says the input already holds them, - and a total that says neither keeps Hermes' rule with the flag unknown. Calls that reported no counts at all got - none from their endpoint, as streamed Claude answers through a LiteLLM proxy, so the turn's counts are unknown - rather than nothing.""" - report = result.usage or {} +# The cost statuses under which Hermes' figure is a price; under any other (``unknown``), its 0.0 means unpriced. +_PRICED = ("estimated", "actual") - def count(name: str) -> int | None: - value = report.get(name) - return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None - cost = report.get("estimated_cost_usd") - details = {key: report[key] for key in ("api_calls", "reasoning_tokens", "cost_status", "turn_exit_reason") +def _count(report: dict, name: str) -> int | None: + value = report.get(name) + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def _usage(result) -> Usage: + """The turn's tool calls and token counts, with ``input_tokens`` counting the cache: its main model calls as + Hermes' usage file counts them, its auxiliary calls (title generation, compression) as the file's ``auxiliary`` + block counts them, and a delegate_task child's calls, which the file leaves out, from their events. Hermes' + ``input_tokens`` is the prompt less its cache reads and writes, and its total counts them, so they're added back; a + total of the input and output alone says the input already holds them, and a total that says neither keeps + Hermes' rule with the flag unknown. Calls that reported no counts at all got none from their endpoint, as streamed + Claude answers through a LiteLLM proxy once did, so the turn's counts are unknown rather than nothing. A turn killed + before it wrote its file counts the calls it made from their events, and one more call as unreported: the one in + flight, or its title call. Each part whose calls reported no counts is counted in ``unreported_calls`` while the + others' counts are kept. Hermes' cost is kept only when it priced every call the turn made.""" + if result.killed_calls is not None: + return _counted_calls(result.tool_call_count, result.killed_calls, unreported=1) + report = result.usage or {} + aux = report.get("auxiliary") if isinstance(report.get("auxiliary"), dict) else {} + side = result.side_calls + calls = (report.get("total_including_auxiliary") or {}).get("api_calls", report.get("api_calls")) + details = {key: report[key] for key in ("reasoning_tokens", "cost_status", "turn_exit_reason") if report.get(key) is not None} - counts = {name: count(name) for name in ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", - "cache_write_tokens")} - if (count("api_calls") or 0) > 0 and not any(counts.values()): - return Usage(tool_call_count=result.tool_call_count, - provider_details={**details, "unreported_calls": details["api_calls"]}) + if calls is not None or side is not None: + details["api_calls"] = (calls or 0) + (side.calls if side else 0) + counts = {name: _count(report, name) for name in ("input_tokens", "output_tokens", "total_tokens", + "cache_read_tokens", "cache_write_tokens")} + unreported = side.unreported if side is not None else 0 + if (_count(report, "api_calls") or 0) > 0 and not any(counts.values()): + unreported += _count(report, "api_calls") + counts = dict.fromkeys(counts) + # Hermes' auxiliary counts are its canonical ones: input without the cache. + extra = {"input": sum(_count(aux, key) or 0 for key in ("input_tokens", "cache_read_tokens", "cache_write_tokens")), + "output": _count(aux, "output_tokens") or 0, "read": _count(aux, "cache_read_tokens") or 0, + "write": _count(aux, "cache_write_tokens") or 0} + if (_count(aux, "api_calls") or 0) > 0 and not (extra["input"] or extra["output"]): + unreported += _count(aux, "api_calls") + if side is not None: + extra = {"input": extra["input"] + side.input_tokens, "output": extra["output"] + side.output_tokens, + "read": extra["read"] + side.cache_read_tokens, "write": extra["write"] + side.cache_write_tokens} + if unreported: + details["unreported_calls"] = unreported read, write = counts["cache_read_tokens"], counts["cache_write_tokens"] - input_tokens, output, total = counts["input_tokens"], counts["output_tokens"] or 0, counts["total_tokens"] + input_tokens, output, total = counts["input_tokens"], counts["output_tokens"], counts["total_tokens"] cached = (read or 0) + (write or 0) includes_cache = None if input_tokens is not None: - if cached == 0 or total == input_tokens + output: + if cached == 0 or total == input_tokens + (output or 0): includes_cache = True else: - includes_cache = True if total == input_tokens + output + cached else None + includes_cache = True if total == input_tokens + (output or 0) + cached else None input_tokens += cached + input_tokens += extra["input"] + output = None if output is None else output + extra["output"] + total = None if total is None else total + extra["input"] + extra["output"] + read = None if read is None else read + extra["read"] + write = None if write is None else write + extra["write"] + elif extra["input"] or extra["output"]: + input_tokens, output, total = extra["input"], extra["output"], extra["input"] + extra["output"] + read, write, includes_cache = extra["read"], extra["write"], True + elif unreported: + return Usage(tool_call_count=result.tool_call_count, provider_details=details) + cost = report.get("estimated_cost_usd") + if aux: + cost = (report.get("total_including_auxiliary") or {}).get("estimated_cost_usd") + # Hermes' cost leaves out a delegate_task child's calls, so a turn with some has none. + priced = (report.get("cost_status") in _PRICED and isinstance(cost, (int, float)) and not isinstance(cost, bool) + and not unreported and not (side and side.calls)) return Usage( tool_call_count=result.tool_call_count, input_tokens=input_tokens, - output_tokens=counts["output_tokens"], + output_tokens=output, total_tokens=total, cache_read_tokens=read, cache_write_tokens=write, input_includes_cache=includes_cache, - cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None, + cost_usd=cost if priced and cost >= 0 else None, provider_details=details, ) +def _counted_calls(tool_calls: int, counted: CallCounts, unreported: int = 0) -> Usage: + """Calls counted from their events: their counts, those without usage and ``unreported`` more counted apart.""" + details = {"api_calls": counted.calls, "unreported_calls": counted.unreported + unreported} + if counted.calls == counted.unreported: + return Usage(tool_call_count=tool_calls, provider_details=details) + return Usage(tool_call_count=tool_calls, input_tokens=counted.input_tokens, output_tokens=counted.output_tokens, + total_tokens=counted.input_tokens + counted.output_tokens, + cache_read_tokens=counted.cache_read_tokens, cache_write_tokens=counted.cache_write_tokens, + input_includes_cache=True, provider_details=details) + + def _replace_workspace(staged: Path, then: Callable[[], None] | None = None) -> None: """Swap staged in for the workspace's contents. When then() raises after the swap, the previous contents come back.""" diff --git a/src/agent_env/agents/agents/hermes/agentenv_plugin/__init__.py b/src/agent_env/agents/agents/hermes/agentenv_plugin/__init__.py index f12aa0ef3..a8aabf368 100644 --- a/src/agent_env/agents/agents/hermes/agentenv_plugin/__init__.py +++ b/src/agent_env/agents/agents/hermes/agentenv_plugin/__init__.py @@ -23,7 +23,8 @@ "post_tool_call": ("session_id", "turn_id", "tool_call_id", "tool_name", "result", "status", "duration_ms", "error_type", "error_message"), # ``response`` is Hermes' sanitized record of the model's answer: its assistant message, tool calls and usage. - "post_api_request": ("session_id", "turn_id", "api_call_count", "model", "finish_reason", "response"), + # ``usage`` is the call's token counts, normalized (``prompt_tokens`` counts the cache), never truncated. + "post_api_request": ("session_id", "turn_id", "api_call_count", "model", "finish_reason", "response", "usage"), "api_request_error": ("session_id", "turn_id", "api_call_count", "model", "status_code", "reason", "retryable", "error"), "post_llm_call": ("session_id", "turn_id", "assistant_response"), diff --git a/src/agent_env/agents/agents/hermes/hermes_bridge.py b/src/agent_env/agents/agents/hermes/hermes_bridge.py index acc5cd9ae..a52626bc9 100644 --- a/src/agent_env/agents/agents/hermes/hermes_bridge.py +++ b/src/agent_env/agents/agents/hermes/hermes_bridge.py @@ -62,6 +62,29 @@ def profile_dir(context_id: str) -> Path: return HERMES_ROOT / "profiles" / f"ctx-{hashlib.sha1(context_id.encode()).hexdigest()[:16]}" +@dataclass +class CallCounts: + """Model calls' counts as their post_api_request events give them, with ``input_tokens`` counting the cache; a call + whose event carried no usage is counted in ``unreported``.""" + + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_tokens: int = 0 + cache_write_tokens: int = 0 + calls: int = 0 + unreported: int = 0 + + def add(self, usage: object) -> None: + self.calls += 1 + if not isinstance(usage, dict) or not (usage.get("prompt_tokens") or usage.get("output_tokens")): + self.unreported += 1 + return + self.input_tokens += int(usage.get("prompt_tokens") or 0) + self.output_tokens += int(usage.get("output_tokens") or 0) + self.cache_read_tokens += int(usage.get("cache_read_tokens") or 0) + self.cache_write_tokens += int(usage.get("cache_write_tokens") or 0) + + @dataclass class HermesResult: response: str @@ -72,6 +95,10 @@ class HermesResult: error_code: str | None = None structured_output: object | None = None usage: dict = field(default_factory=dict) + # The model calls of other turns in the process, a delegate_task child's, which the usage file leaves out. + side_calls: CallCounts | None = None + # A turn killed before it wrote its usage file: the calls it made, counted from their events. + killed_calls: CallCounts | None = None def validate_output_format(output_format: dict) -> dict: @@ -178,6 +205,9 @@ def build_config( "providers": {PROVIDER: provider}, "skills": {"external_dirs": [str(skills_dir)]}, "plugins": {"enabled": [_PLUGIN_NAME]}, + # Hermes' review fork replays the turn on the main model in a thread the turn's exit kills before it reports + # its calls, which the proxy bills anyway, as Hermes turns it off for its own one-shot turns. + "auxiliary": {"background_review": {"enabled": False}}, } if mcp_servers: servers: dict[str, dict] = {} @@ -360,7 +390,8 @@ async def run_hermes( _kill_cli(proc) await _reap(proc) return HermesResult(response="", tool_call_count=parser.tool_call_count, session_id=parser.session_id, - trajectory=parser.trajectory, error=f"hermes timed out after {timeout_s}s") + trajectory=parser.trajectory, error=f"hermes timed out after {timeout_s}s", + killed_calls=parser.all_calls) except Exception: # A failure while reading, such as an event_callback that raises, must not leave the turn running. _kill_cli(proc) @@ -422,8 +453,10 @@ class _EventParser: """Parse the plugin's JSON event lines one at a time. Every event carries its hook's name as ``type``. ``pre_tool_call`` events are the turn's tool calls, - ``post_llm_call`` carries the final answer, and any event's ``session_id`` is the session. The whole event list is - the trajectory, and each event is handed to ``event_callback`` as it is parsed. + ``post_llm_call`` carries the final answer, and the turn's events' ``session_id`` is the session: the turn is the + one whose ``turn_id`` the first model call carries, as the turn calls the model before it can delegate, and another + turn's events are a delegate_task child's, whose model calls are counted apart. The whole event list is the + trajectory, and each event is handed to ``event_callback`` as it is parsed. """ def __init__(self, event_callback: Callable[[dict], None] | None = None) -> None: @@ -434,6 +467,9 @@ def __init__(self, event_callback: Callable[[dict], None] | None = None) -> None self.final: str | None = None self.errors: list[str] = [] self.error_code: str | None = None + self.turn_id: str | None = None + self.all_calls = CallCounts() + self.side_calls = CallCounts() @property def tool_call_count(self) -> int: @@ -451,9 +487,16 @@ def on_line(self, line: str) -> None: if not isinstance(event, dict): return self.trajectory.append(event) - if isinstance(event.get("session_id"), str) and event["session_id"]: - self.session_id = event["session_id"] kind = event.get("type") + if self.turn_id is None and kind in ("post_api_request", "api_request_error") and event.get("turn_id"): + self.turn_id = event["turn_id"] + theirs = self.turn_id is not None and event.get("turn_id") != self.turn_id + if isinstance(event.get("session_id"), str) and event["session_id"] and not theirs: + self.session_id = event["session_id"] + if kind == "post_api_request": + self.all_calls.add(event.get("usage")) + if theirs: + self.side_calls.add(event.get("usage")) if kind == "pre_tool_call": self.tool_calls.add(str(event.get("tool_call_id") or f"tool-{len(self.tool_calls)}")) elif kind == "post_llm_call" and isinstance(event.get("assistant_response"), str): @@ -487,4 +530,5 @@ def result(self, stdout: bytes, stderr: bytes, returncode: int | None, usage: di error=err, error_code=self.error_code if err else None, usage=usage, + side_calls=self.side_calls if self.side_calls.calls else None, ) diff --git a/src/agent_env/agents/agents/openhands/openhands_agent.py b/src/agent_env/agents/agents/openhands/openhands_agent.py index 43d4f5bdb..7af6f34b4 100644 --- a/src/agent_env/agents/agents/openhands/openhands_agent.py +++ b/src/agent_env/agents/agents/openhands/openhands_agent.py @@ -160,11 +160,11 @@ class Outcome: def turn_usage(calls: list[tuple[Any, bool]], *, responses: int, tool_calls: int, cost: float | None) -> Usage: """A turn's usage from the ``TokenUsage`` OpenHands recorded for each model call, beside whether its LLM speaks OpenAI's Responses API, with ``input_tokens`` counting the cache. A Responses call's input counts it. A Chat - Completions call's may or may not: LiteLLM reads cached tokens from OpenAI's ``prompt_tokens_details``, whose - prompt counts them, and from Anthropic's keys beside a prompt that doesn't, as a LiteLLM proxy sends Claude's. - The cache is added when the prompt is smaller than it, and the turn's flag is left unknown when the counts can't - say. OpenHands records no ``TokenUsage`` for a response without tokens, so the turn's ``responses`` beyond its - calls are counted in ``unreported_calls``.""" + Completions call's comes through LiteLLM in OpenAI's shape, which can't say, so it's read by OpenAI's rule: a + prompt at least as large as its cache counts it, as OpenAI's and a LiteLLM proxy's Claude prompts do, and a smaller + one can't, so the cache is added (as such a proxy sent Claude's before it counted the cache). OpenHands records no + ``TokenUsage`` for a response without tokens, so the turn's ``responses`` beyond its calls are counted in + ``unreported_calls``.""" counted = [_counted(usage.prompt_tokens, usage.completion_tokens, usage.cache_read_tokens, usage.cache_write_tokens, True if responses_api else None) for usage, responses_api in calls] unreported = responses - len(calls) @@ -174,13 +174,11 @@ def turn_usage(calls: list[tuple[Any, bool]], *, responses: int, tool_calls: int def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]: - """A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the - cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0.""" - if counts_cache or read + write == 0: + """A call's input counting its cache, its output, cache read, cache write, and 1, its flag settled: a prompt said + to count the cache (``counts_cache``), or at least as large as it, is the input; a smaller one gets it added.""" + if counts_cache or prompt >= read + write: return prompt, output, read, write, 1 - if prompt < read + write: - return prompt + read + write, output, read, write, 1 - return prompt, output, read, write, 0 + return prompt + read + write, output, read, write, 1 def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage: diff --git a/src/agent_env/agents/agents/pi/pi_agent.py b/src/agent_env/agents/agents/pi/pi_agent.py index 158312c5d..1f14a169d 100644 --- a/src/agent_env/agents/agents/pi/pi_agent.py +++ b/src/agent_env/agents/agents/pi/pi_agent.py @@ -137,6 +137,8 @@ class _Run: session_id: str | None = None events: list[dict[str, Any]] = field(default_factory=list) assistant_messages: list[dict[str, Any]] = field(default_factory=list) + # Each compaction's summary usage: pi makes that call outside the agent loop and reports it only here. + summary_usages: list[dict[str, Any]] = field(default_factory=list) tool_calls: int = 0 @@ -250,20 +252,23 @@ def _record(run: _Run, event: dict[str, Any]) -> None: run.assistant_messages.append(event["message"]) elif kind == "tool_execution_end": run.tool_calls += 1 + elif kind == "compaction_end": + run.summary_usages.append((event.get("result") or {}).get("usage") or {}) if kind not in _UNRECORDED_EVENTS: run.events.append(event) def _usage(run: _Run) -> Usage: - """The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache. A message whose - usage is all zero got none from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted - in ``unreported_calls`` instead of as nothing used.""" + """The turn's usage, summed over its assistant messages and its compactions' summary calls, with ``input_tokens`` + counting the cache. A usage that is all zero got none from its endpoint, as a streamed Claude answer through a + LiteLLM proxy, or a compaction that ended without one, and is counted in ``unreported_calls`` instead of as nothing + used; a message pi ended with an error and no usage was a request the endpoint refused, and counts nothing.""" counted, cost, unreported = [], 0.0, 0 - for message in run.assistant_messages: - usage = message.get("usage") or {} + usages = [(message.get("usage") or {}, message.get("stopReason") == "error") for message in run.assistant_messages] + for usage, refused in usages + [(usage, False) for usage in run.summary_usages]: counts = [int(usage.get(key) or 0) for key in ("input", "output", "cacheRead", "cacheWrite", "totalTokens")] if not any(counts): - unreported += 1 + unreported += not refused continue counted.append(_counted(*counts)) cost += float((usage.get("cost") or {}).get("total") or 0) diff --git a/src/agent_env/agents/agents/pydantic-ai/README.md b/src/agent_env/agents/agents/pydantic-ai/README.md index 2c60390ad..33a3d628e 100644 --- a/src/agent_env/agents/agents/pydantic-ai/README.md +++ b/src/agent_env/agents/agents/pydantic-ai/README.md @@ -53,7 +53,8 @@ Snapshot `load` answers 409 while a task is running, and 400 for an archive that other contexts, which work in the same workspace. When the workspace holds the agent's own `~/.pydantic-ai-a2a`, as an install in a home directory's does, the workspace snapshot leaves it out and `load` keeps it as it is. -Usage is the run's: input, output and cached tokens, model requests and tool calls; no cost. A failed run's error code +Usage is the run's, the runs it delegates to with `delegate_task` included: input, output and cached tokens, model +requests and tool calls; no cost. A failed run's error code says whose failure it was: `pydantic_ai.model_http_` and `pydantic_ai.model_api` for the model endpoint and `pydantic_ai.error` for anything else that stopped the run, such as an MCP server that can't be reached, are `infra_error`s; `pydantic_ai.usage_limit` (the request limit) and `pydantic_ai.unexpected_model_behavior` are diff --git a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py index 0f256a89d..837904878 100644 --- a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py +++ b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py @@ -8,7 +8,7 @@ from agentenv_protocol.a2a_agent import Usage from pydantic_ai import Agent -from pydantic_ai.capabilities import LocalWorkspace +from pydantic_ai.capabilities import Hooks, LocalWorkspace from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UnexpectedModelBehavior, UsageLimitExceeded from pydantic_ai.mcp import MCPToolset from pydantic_ai.messages import BinaryContent, ModelMessage, ModelMessagesTypeAdapter, ModelResponse, UserContent @@ -49,11 +49,20 @@ def _settings(turn: Turn) -> ModelSettings: return settings -def _agent(turn: Turn) -> Agent: +def _agent(turn: Turn, responses: list[ModelResponse]) -> Agent: + """The turn's agent, which puts every model response it gets in ``responses``: its own, and those of the runs it + delegates to with delegate_task, which run this agent again with messages of their own, so they're never in the + turn's messages. Listed last, the recorder is the first to see each response.""" + + async def keep(_ctx: Any, /, *, request_context: Any, response: ModelResponse) -> ModelResponse: + responses.append(response) + return response + # The agent runs in a sandbox of its own, so its file tools reach the files attached outside the workspace. capabilities: list[Any] = [LocalWorkspace(turn.workspace), Coder(unrestricted_filesystem=True)] if turn.skill_names: capabilities.append(Skills(turn.skills_dir, include=turn.skill_names)) + capabilities.append(Hooks(after_model_request=keep)) toolsets = [ MCPToolset(registration["url"], id=name, headers=dict(registration.get("headers") or {})) for name, registration in turn.mcp_servers.items() @@ -85,10 +94,10 @@ def _events(messages: Sequence[ModelMessage]) -> list[dict[str, Any]]: return ModelMessagesTypeAdapter.dump_python(list(messages), mode="json") -def _usage(messages: Sequence[ModelMessage], usage: RunUsage) -> Usage: - """The turn's usage from each of its model responses', and its tool calls and requests from the run's.""" - return turn_usage((message.usage for message in messages if isinstance(message, ModelResponse)), - tool_calls=usage.tool_calls, requests=usage.requests) +def _usage(responses: Sequence[ModelResponse], usage: RunUsage) -> Usage: + """The turn's usage from each of its model responses', delegated runs' included, and its tool calls and requests + from the run's, which counts the delegated runs' too.""" + return turn_usage((response.usage for response in responses), tool_calls=usage.tool_calls, requests=usage.requests) def _failure(exc: Exception, usage: Usage) -> Outcome: @@ -136,8 +145,9 @@ async def _run(self, turn: Turn) -> Outcome: prompt: list[UserContent] = [turn.text] for image in turn.images: prompt.append(BinaryContent(data=await asyncio.to_thread(image.path.read_bytes), media_type=image.media_type)) - agent = _agent(turn) - started = recorded = len(history) + responses: list[ModelResponse] = [] + agent = _agent(turn, responses) + recorded = len(history) messages: list[ModelMessage] = list(history) usage = RunUsage() try: @@ -151,14 +161,14 @@ async def _run(self, turn: Turn) -> Outcome: usage = run.usage result = run.result except asyncio.CancelledError: - turn.on_usage(_usage(messages[started:], usage)) + turn.on_usage(_usage(responses, usage)) raise except Exception as exc: # every failure is the task's, typed by whose it was - return _failure(exc, _usage(messages[started:], usage)) + return _failure(exc, _usage(responses, usage)) messages = result.all_messages() for event in _events(messages[recorded:]): turn.on_event(event) # A turn whose run finished is saved whole even if it is cancelled meanwhile, before its locks are let go. await to_completion(_save, turn, messages) return Outcome(result.output.strip() or "The agent finished without a reply.", - _usage(messages[started:], result.usage)) + _usage(responses, result.usage)) diff --git a/tst/integration/cli/run_bundle_openhands_test.py b/tst/integration/cli/run_bundle_openhands_test.py index 6957cf1f1..cb81dcdc9 100644 --- a/tst/integration/cli/run_bundle_openhands_test.py +++ b/tst/integration/cli/run_bundle_openhands_test.py @@ -260,13 +260,13 @@ async def steps(context): assert {outcome["state"] for outcome in outcomes.values()} == {"completed"}, outcomes usage = {name: Usage.from_dict(outcome["data"]["usage"]) for name, outcome in outcomes.items()} - # GPT's prompt is as large as its cache, so whether it counts it is left unknown. + # GPT's prompt is at least as large as its cache, which OpenAI's rule takes it to count, as it does. assert usage == { "claude": Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, provider_details={"reasoning_tokens": 0}), "gpt": Usage(tool_call_count=0, input_tokens=2303, output_tokens=2, total_tokens=2305, cache_read_tokens=2048, - cache_write_tokens=0, input_includes_cache=None, provider_details={"reasoning_tokens": 0}), + cache_write_tokens=0, input_includes_cache=True, provider_details={"reasoning_tokens": 0}), "none": Usage(tool_call_count=0, provider_details={"reasoning_tokens": 0, "unreported_calls": 1}), }, outcomes diff --git a/tst/unit/agents/browser_use/fake_agent.py b/tst/unit/agents/browser_use/fake_agent.py index f638d0d22..35beeb635 100644 --- a/tst/unit/agents/browser_use/fake_agent.py +++ b/tst/unit/agents/browser_use/fake_agent.py @@ -1,5 +1,6 @@ """Stands in for ``agent.py``: records the turn it's given, and its environment, to ``FAKE_TURN_RECORD`` and answers -as the run ``FAKE_SCENARIO`` names (``ok``, ``max_steps``, ``crash``, ``hang``).""" +as the run ``FAKE_SCENARIO`` names (``ok``, ``max_steps``, ``crash``, ``hang``; ``cut``, ``mixed`` and ``refused`` for +requests that got no response; ``follow_up`` for a context's third turn).""" import json import os @@ -19,15 +20,26 @@ # A library's own stdout, before agent.py's line. print("{ not json") print(json.dumps({"level": "info", "msg": "connected"})) +TURN_MESSAGES = [ + {"kind": "request", "parts": [{"part_kind": "user-prompt", "content": turn["prompt"]}]}, + {"kind": "response", "parts": [{"part_kind": "tool-call", "tool_name": "python", "args": {"code": "page_info()"}}]}, + {"kind": "response", "parts": [{"part_kind": "tool-call", "tool_name": "finish", "args": {"result": "x"}}]}, +] +USAGE = {"input": 100, "output": 20, "cache_read": 60, "cache_write": 10, "requests": 3} +messages, prior, usage, calls, cost = TURN_MESSAGES, 0, USAGE, {"sent": 3, "refused": 0}, None +if scenario == "cut": # browser-use-cut-request: the run's 1 s budget cut its only request; the proxy still answered + messages, usage, calls, cost = TURN_MESSAGES[:1], dict.fromkeys(USAGE, 0), {"sent": 1, "refused": 0}, 0.0 +elif scenario == "mixed": + usage, calls = {**USAGE, "requests": 2}, {"sent": 3, "refused": 0} +elif scenario == "refused": + usage, calls = {**USAGE, "requests": 2}, {"sent": 3, "refused": 1} +elif scenario == "follow_up": # two earlier turns of three messages each, then this turn's + messages, prior = TURN_MESSAGES * 3, 6 print(json.dumps({"type": "result", "result": { - "status": "completed" if scenario == "ok" else scenario, - "text": "The top story." if scenario == "ok" else "", - "success": True if scenario == "ok" else None, "error": None, - "steps": 3, "model": turn["model"], "cost_usd": None, - "usage": {"input": 100, "output": 20, "cache_read": 60, "cache_write": 10, "requests": 3}, - "messages": [ - {"kind": "request", "parts": [{"part_kind": "user-prompt", "content": turn["prompt"]}]}, - {"kind": "response", "parts": [{"part_kind": "tool-call", "tool_name": "python", "args": {"code": "page_info()"}}]}, - {"kind": "response", "parts": [{"part_kind": "tool-call", "tool_name": "finish", "args": {"result": "x"}}]}, - ], + "status": "completed" if scenario in ("ok", "mixed", "refused", "follow_up") else + "timeout" if scenario == "cut" else scenario, + "text": "" if scenario in ("max_steps", "cut") else "The top story.", + "success": None if scenario in ("max_steps", "cut") else True, "error": None, + "steps": 0 if scenario == "cut" else 3, "model": turn["model"], "cost_usd": cost, + "usage": usage, "messages": messages, "prior": prior, "calls": calls, }})) diff --git a/tst/unit/agents/browser_use/test_browser_use_agent.py b/tst/unit/agents/browser_use/test_browser_use_agent.py index 848f2df85..4cc88eba0 100644 --- a/tst/unit/agents/browser_use/test_browser_use_agent.py +++ b/tst/unit/agents/browser_use/test_browser_use_agent.py @@ -6,9 +6,12 @@ from typing import Any import pytest +from agentenv_protocol.a2a_agent import Usage from starlette.testclient import TestClient import agent_env +from agent_env.config.model_pricing import litellm_prices +from agent_env.task_step.usage import RoleUsage AGENT_DIR = Path(agent_env.__file__).parent / "agents" / "agents" / "browser-use" # By its path: another agent's a2a_server, imported by the same name, may be loaded already. @@ -125,6 +128,40 @@ def test_a_run_that_stops_early_fails_with_its_reason(client: TestClient, monkey assert _reply(task)[1]["error_code"] == "browser_use.max_steps" +def _usage_of(client: TestClient, monkeypatch: pytest.MonkeyPatch, scenario: str) -> dict[str, Any]: + monkeypatch.setenv("FAKE_SCENARIO", scenario) + return _reply(_send(client, scenario))[1]["usage"] + + +def test_a_request_cut_before_its_response_is_unknown_not_zero(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + """browser-use-cut-request, recorded on 2026-11-20: the run's budget cut its only request, which the proxy still + answered (2,739 in, 6 out), and browser-use counted no response; the turn's usage is unknown, not $0.""" + usage = _usage_of(client, monkeypatch, "cut") + role = RoleUsage("solver", "claude-haiku-4-5-20251001") + role.add(Usage.from_dict(usage)) + entry = role.entry(litellm_prices()) + + assert "input_tokens" not in usage and "cost_usd" not in usage + assert usage["provider_details"]["unreported_calls"] == 1 + assert entry["partial"] is True and entry["cost_usd"] is None and entry["unreported_calls"] == 1 + + +def test_requests_without_a_counted_response_mark_the_turn_and_refused_ones_dont( + client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + mixed = _usage_of(client, monkeypatch, "mixed") + refused = _usage_of(client, monkeypatch, "refused") + + assert mixed["input_tokens"] == 100 and mixed["provider_details"]["unreported_calls"] == 1 + assert refused["input_tokens"] == 100 and "unreported_calls" not in refused["provider_details"] + + +def test_a_turn_counts_its_own_tool_calls_not_the_conversations(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + """browser-use's run returns the whole conversation's messages; the turn's are those after the earlier turns'.""" + assert _usage_of(client, monkeypatch, "follow_up")["tool_call_count"] == 2 + + def test_effort_outside_browser_uses_is_refused_with_the_config(client: TestClient) -> None: assert client.post("/ext/agent-config", json={"effort": "xhigh"}).status_code == 400 diff --git a/tst/unit/agents/hermes/conftest.py b/tst/unit/agents/hermes/conftest.py index a10588f60..10bc586a9 100644 --- a/tst/unit/agents/hermes/conftest.py +++ b/tst/unit/agents/hermes/conftest.py @@ -50,7 +50,8 @@ async def send(client: httpx.AsyncClient, parts: str | list[dict], context_id: s def result(**fields) -> SimpleNamespace: """What run_hermes returns, with a finished turn's defaults.""" defaults = {"response": "done", "error": None, "error_code": None, "session_id": "s-1", "trajectory": [], - "tool_call_count": 0, "structured_output": None, "usage": {}} + "tool_call_count": 0, "structured_output": None, "usage": {}, "side_calls": None, + "killed_calls": None} return SimpleNamespace(**{**defaults, **fields}) diff --git a/tst/unit/agents/hermes/test_message_contract.py b/tst/unit/agents/hermes/test_message_contract.py index 5501d0963..318202a1e 100644 --- a/tst/unit/agents/hermes/test_message_contract.py +++ b/tst/unit/agents/hermes/test_message_contract.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import json import os import shutil import stat @@ -251,10 +252,117 @@ def test_a_turns_usage_adds_the_cache_back_to_the_input(): assert a2a_server._usage(result(usage=report, tool_call_count=1)).to_dict() == { "tool_call_count": 1, "input_tokens": 21246, "output_tokens": 48, "total_tokens": 21294, - "cache_read_tokens": 10496, "cache_write_tokens": 0, "input_includes_cache": True, "cost_usd": 0.0, + "cache_read_tokens": 10496, "cache_write_tokens": 0, "input_includes_cache": True, "provider_details": {"api_calls": 2, "reasoning_tokens": 0, "cost_status": "unknown"}} +def test_hermes_cost_is_kept_only_when_it_priced_the_calls(): + """Hermes reports 0.0 with ``cost_status`` unknown when it couldn't price a call; that isn't a price.""" + report = {"input_tokens": 100, "output_tokens": 10, "total_tokens": 110, "cache_read_tokens": 0, + "cache_write_tokens": 0, "api_calls": 1} + + unknown = a2a_server._usage(result(usage={**report, "estimated_cost_usd": 0.0, "cost_status": "unknown"})) + estimated = a2a_server._usage(result(usage={**report, "estimated_cost_usd": 0.0123, "cost_status": "estimated"})) + + assert unknown.cost_usd is None and estimated.cost_usd == 0.0123 + + +def test_a_turns_usage_counts_its_auxiliary_calls(): + """hermes-baseline-gpt, recorded on 2026-11-20 through Scale's LiteLLM proxy: two main calls and a title-generation + call, which Hermes reports only in its usage file's auxiliary block; the proxy saw 21,523 in and 64 out.""" + report = {"input_tokens": 10739, "output_tokens": 54, "total_tokens": 21289, "cache_read_tokens": 10496, + "cache_write_tokens": 0, "api_calls": 2, "estimated_cost_usd": 0.0, "cost_status": "unknown", + "auxiliary": {"api_calls": 1, "input_tokens": 288, "output_tokens": 10, "cache_read_tokens": 0, + "cache_write_tokens": 0, "reasoning_tokens": 0, "estimated_cost_usd": 0.0, + "total_tokens": 298, "by_task": {"title_generation": {"api_calls": 1}}}, + "total_including_auxiliary": {"estimated_cost_usd": 0.0, "total_tokens": 21587, "api_calls": 3}} + + usage = a2a_server._usage(result(usage=report)) + + assert (usage.input_tokens, usage.output_tokens, usage.cache_read_tokens) == (21523, 64, 10496) + assert usage.provider_details["api_calls"] == 3 and usage.cost_usd is None + + +def _events(*events): + parser = hermes_bridge._EventParser() + for event in events: + parser.on_line(json.dumps(event)) + return parser + + +def _call(turn, session, prompt, output): + return {"type": "post_api_request", "turn_id": turn, "session_id": session, + "usage": {"input_tokens": prompt, "prompt_tokens": prompt, "output_tokens": output, + "cache_read_tokens": 0, "cache_write_tokens": 0}} + + +def test_a_delegate_task_childs_calls_are_counted_and_the_session_stays_the_turns(): + """hermes-subagent-claude, recorded on 2026-11-20: the turn's two calls (26,616 in, 86 out, as its usage file + counts them) and its title call (456, 17) beside a child's two (9,161 and 9,251 in), which the usage file leaves + out; the proxy saw 45,484 in and 172 out.""" + parser = _events({"type": "on_session_start", "session_id": "parent"}, _call("A", "parent", 13081, 82), + {"type": "on_session_start", "session_id": "child"}, _call("B", "child", 9161, 60), + _call("B", "child", 9251, 9), _call("A", "parent", 13535, 4)) + report = {"input_tokens": 26616, "output_tokens": 86, "total_tokens": 26702, "cache_read_tokens": 0, + "cache_write_tokens": 0, "api_calls": 2, "cost_status": "unknown", + "auxiliary": {"api_calls": 1, "input_tokens": 456, "output_tokens": 17, "cache_read_tokens": 0, + "cache_write_tokens": 0, "total_tokens": 473}, + "total_including_auxiliary": {"api_calls": 3}} + + turn = parser.result(b"", b"", 0, report) + usage = a2a_server._usage(turn) + + assert turn.session_id == "parent" and parser.session_id == "parent" + assert (usage.input_tokens, usage.output_tokens, usage.provider_details["api_calls"]) == (45484, 172, 5) + + +def test_a_priced_turn_with_a_delegate_task_child_has_no_cost(): + """Hermes prices only the calls its usage file counts, which leave out a child's, so its figure is short.""" + parser = _events(_call("A", "parent", 13081, 82), _call("B", "child", 9161, 60), _call("A", "parent", 13535, 4)) + report = {"input_tokens": 26616, "output_tokens": 86, "total_tokens": 26702, "cache_read_tokens": 0, + "cache_write_tokens": 0, "api_calls": 2, "estimated_cost_usd": 0.0271, "cost_status": "estimated"} + + alone = a2a_server._usage(result(usage=report)) + delegated = a2a_server._usage(parser.result(b"", b"", 0, report)) + + assert (alone.cost_usd, delegated.cost_usd, delegated.input_tokens) == (0.0271, None, 26616 + 9161) + + +def test_calls_that_reported_counts_are_kept_beside_those_that_reported_none(): + """A streamed main answer without usage leaves the turn's title call and a child's call counted, and only the main + calls unreported.""" + parser = _events(_call("A", "parent", 0, 0), _call("B", "child", 9161, 60)) + report = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cache_read_tokens": 0, + "cache_write_tokens": 0, "api_calls": 2, "estimated_cost_usd": 0.0, "cost_status": "estimated", + "auxiliary": {"api_calls": 1, "input_tokens": 456, "output_tokens": 17, "cache_read_tokens": 0, + "cache_write_tokens": 0, "total_tokens": 473}, + "total_including_auxiliary": {"api_calls": 3}} + + usage = a2a_server._usage(parser.result(b"", b"", 0, report)) + + assert usage.to_dict() == { + "tool_call_count": 0, "input_tokens": 456 + 9161, "output_tokens": 17 + 60, "total_tokens": 456 + 9161 + 17 + 60, + "cache_read_tokens": 0, "cache_write_tokens": 0, "input_includes_cache": True, + "provider_details": {"api_calls": 4, "unreported_calls": 2, "cost_status": "estimated"}} + + +def test_a_killed_turn_keeps_the_calls_it_made_as_a_lower_bound(): + """hermes-timeout-claude, recorded on 2026-11-20: two calls (13,073 and 13,156 in, 53 and 81 out) before the turn + ran out of time, killed before it wrote its usage file.""" + parser = _events(_call("A", "s", 13073, 53), _call("A", "s", 13156, 81)) + + usage = a2a_server._usage(result(usage={}, killed_calls=parser.all_calls)) + + assert (usage.input_tokens, usage.output_tokens, usage.input_includes_cache) == (26229, 134, True) + assert usage.provider_details == {"api_calls": 2, "unreported_calls": 1} + + +def test_the_profile_turns_hermes_review_fork_off(): + config = hermes_bridge.build_config("m", None, {}, None, Path("/skills"), environ={"LITELLM_BASE_URL": "http://x"}) + + assert config["auxiliary"] == {"background_review": {"enabled": False}} + + def test_a_turn_whose_calls_reported_no_usage_has_unknown_counts(): """Claude's streamed answers through a LiteLLM proxy carry no usage, so Hermes reports zeros for them.""" report = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cache_read_tokens": 0, diff --git a/tst/unit/agents/openhands/test_openhands_agent.py b/tst/unit/agents/openhands/test_openhands_agent.py index a686a44a0..790aeeaee 100644 --- a/tst/unit/agents/openhands/test_openhands_agent.py +++ b/tst/unit/agents/openhands/test_openhands_agent.py @@ -17,6 +17,9 @@ from agentenv_protocol.a2a_agent import Usage from starlette.testclient import TestClient +from agent_env.config.model_pricing import litellm_prices +from agent_env.task_step.usage import RoleUsage + from tst.unit.agents.openhands.agent import openhands_agent OpenHandsAgent = openhands_agent.OpenHandsAgent @@ -703,11 +706,27 @@ def test_a_turns_usage_counts_the_cache_in_its_input() -> None: def test_a_turns_usage_is_unknown_where_no_usage_came_back() -> None: """A response that carried no usage, which OpenHands keeps no ``TokenUsage`` for, is counted apart, and a Chat - Completions prompt as large as its cache may or may not have counted it.""" + Completions prompt as large as its cache counts it, by OpenAI's rule.""" unreported = openhands_agent.turn_usage([], responses=1, tool_calls=0, cost=None) - ambiguous = openhands_agent.turn_usage([(_call(5000, 10, read=4000), False)], responses=2, tool_calls=2, cost=None) + counted = openhands_agent.turn_usage([(_call(5000, 10, read=4000), False)], responses=2, tool_calls=2, cost=None) assert unreported == Usage(tool_call_count=0, provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) - assert ambiguous == Usage(tool_call_count=2, input_tokens=5000, output_tokens=10, total_tokens=5010, - cache_read_tokens=4000, cache_write_tokens=0, input_includes_cache=None, - provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) + assert counted == Usage(tool_call_count=2, input_tokens=5000, output_tokens=10, total_tokens=5010, + cache_read_tokens=4000, cache_write_tokens=0, input_includes_cache=True, + provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) + + +def test_a_claude_turn_through_a_proxy_that_counts_the_cache_is_priced_once() -> None: + """openhands-baseline-claude, recorded on 2026-11-20 through Scale's LiteLLM proxy, whose Claude prompt counts the + cache: the turn's cost is what the proxy's calls cost, not the cache charged twice.""" + usage = openhands_agent.turn_usage([(_call(10381, 215, write=10371), False), + (_call(10813, 65, read=10371, write=428), False)], + responses=2, tool_calls=2, cost=0.01595985) + role = RoleUsage("solver", "claude-haiku-4-5-20251001") + role.add(usage) + entry = role.entry(litellm_prices()) + + assert (usage.input_tokens, usage.cache_read_tokens, usage.cache_write_tokens, usage.input_includes_cache) == ( + 21194, 10371, 10799, True) + assert abs(entry["cost_usd"] - 0.01595985) < 1e-9 and entry["uncached_input_tokens"] == 24 + assert "cost_assumed" not in entry diff --git a/tst/unit/agents/pi/test_pi_agent.py b/tst/unit/agents/pi/test_pi_agent.py index 21c92389d..53e3ca63c 100644 --- a/tst/unit/agents/pi/test_pi_agent.py +++ b/tst/unit/agents/pi/test_pi_agent.py @@ -108,6 +108,38 @@ def test_usage_adds_the_cache_back_to_the_input() -> None: } +def _compacted_run(summary: dict[str, Any] | None) -> pi_agent._Run: + """pi-compaction-gpt as pi reported it on 2026-11-20 through Scale's LiteLLM proxy: a call that ran 18 tools, an + auto-compaction whose summary call (329 in, 156 out) pi reports only in its compaction_end, then the answer.""" + run = pi_agent._Run() + events = [{"type": "session", "id": "s"}, + {"type": "message_end", "message": {"role": "assistant", "usage": { + "input": 195, "output": 1150, "cacheRead": 1024, "cacheWrite": 0, "totalTokens": 2369, + "cost": {"total": 0}}}}, + *({"type": "tool_execution_end"} for _ in range(18)), + {"type": "compaction_start", "reason": "threshold"}, + {"type": "compaction_end", "result": summary and {"summary": "...", "usage": summary}}, + {"type": "message_end", "message": {"role": "assistant", "usage": { + "input": 326471, "output": 1, "cacheRead": 0, "cacheWrite": 0, "totalTokens": 326472}}}] + for event in events: + pi_agent._record(run, event) + return run + + +def test_usage_counts_a_compactions_summary_call() -> None: + """The proxy answered three calls: 328,019 input (1,024 cached) and 1,307 output in all.""" + run = _compacted_run({"input": 329, "output": 156, "cacheRead": 0, "cacheWrite": 0, "totalTokens": 485, + "cost": {"total": 0}}) + + assert pi_agent._usage(run).to_dict() == { + "tool_call_count": 18, "input_tokens": 328019, "output_tokens": 1307, "total_tokens": 329326, + "cache_read_tokens": 1024, "cache_write_tokens": 0, "input_includes_cache": True} + + +def test_a_compaction_that_ended_without_usage_is_unreported() -> None: + assert pi_agent._usage(_compacted_run(None)).provider_details == {"unreported_calls": 1} + + def test_usage_follows_what_each_messages_total_says_its_input_holds() -> None: """A total of the input and output alone says pi counted the cache in the input already; one that is neither sum keeps pi's rule, unconfirmed.""" @@ -134,6 +166,17 @@ def test_a_call_that_reported_no_usage_is_unknown_not_zero() -> None: "cache_write_tokens": 0, "input_includes_cache": True, "provider_details": {"unreported_calls": 1}} +def test_a_request_the_endpoint_refused_counts_nothing() -> None: + """As in pi-retry-faults on 2026-11-20: pi ends a request the proxy refused with a 500 as an error message with no + usage, then retries it. The refused request wasn't billed; the retry was answered.""" + refused = {**_message(0, 0, 0, 0), "stopReason": "error", "errorMessage": "500 Internal Server Error"} + run = pi_agent._Run(assistant_messages=[refused, _message(1109, 21, 0, 1130)]) + + assert pi_agent._usage(run).to_dict() == { + "tool_call_count": 0, "input_tokens": 1109, "output_tokens": 21, "total_tokens": 1130, "cache_read_tokens": 0, + "cache_write_tokens": 0, "input_includes_cache": True} + + def test_a_context_keeps_one_pi_session(client: TestClient, record: Path) -> None: _send(client, [{"kind": "text", "text": "one"}], context_id="ctx/a b") first = json.loads(record.read_text())["argv"]