Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 39 additions & 9 deletions packages/agentenv-protocol/tests/test_browser_recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,17 +69,45 @@ async def upload(self, path, source):
return Uploader()

def colors(self, folder: str) -> list[str]:
"""The colour of every frame of the video under ``folder``, in order, by its strongest channel."""
names = {RED: "red", BLUE: "blue", GREEN: "green", YELLOW: "yellow", (255, 255, 255): "white"}
"""The colour of every frame of the video under ``folder``, in order."""
found = []
for path in sorted(p for p in self.by_path if p.startswith(f"{folder}/video/") and p.endswith(".mp4")):
with av.open(io.BytesIO(self.by_path[path])) as container:
for frame in container.decode(video=0):
pixel = frame.to_image().getpixel((frame.width // 2, frame.height // 2))
found.append(names[min(names, key=lambda c: sum((a - b) ** 2 for a, b in zip(c, pixel)))])
found += [_color_name(frame.to_image()) for frame in container.decode(video=0)]
return found


def _color_name(image: Image.Image) -> str:
"""The named colour nearest the middle of ``image``."""
names = {RED: "red", BLUE: "blue", GREEN: "green", YELLOW: "yellow", (255, 255, 255): "white"}
pixel = image.convert("RGB").getpixel((image.width // 2, image.height // 2))
return names[min(names, key=lambda c: sum((a - b) ** 2 for a, b in zip(c, pixel)))]


def _taken(recordings: BrowserRecordings) -> list[str]:
"""The colour of each frame the one recording's video takes from now on, as it takes it."""
[recording] = recordings.every()
stream, taken = recording.stream, []
latest = stream._latest

async def noted() -> Image.Image:
image = await latest()
taken.append(_color_name(image))
return image
stream._latest = noted
return taken


async def _until_taken(taken: list[str], color: str, timeout_s: float = 10) -> None:
"""Until the video has encoded a frame of ``color``, however slowly a loaded machine records: until it has taken
the frame after it, which it takes only once it has encoded the one before."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout_s
while color not in taken or len(taken) == taken.index(color) + 1:
assert loop.time() < deadline, f"no {color} frame encoded in {timeout_s} s: {taken}"
await asyncio.sleep(0.02)


class Browser:
"""A browser's pipe, as recordings ask it: each tab attached on its own session, which a screencast paints in the
tab's colour unless it's ``silent``, as a tab behind another is, hidden, and a capture answers in it; every other
Expand Down Expand Up @@ -213,13 +241,15 @@ async def test_the_video_follows_an_agents_tab_a_popup_it_opens_and_goes_back_wh
pipe.open("T3", YELLOW)
pipe.focused = "T0"
started = await recordings.start("run:a", _grant(), network=False)
await asyncio.sleep(0.3)
taken = _taken(recordings)
await _until_taken(taken, "green")
_acted(recordings.log, "T3")
await asyncio.sleep(0.3)
await _until_taken(taken, "yellow")
pipe.open("T4", RED, opener="T3")
await asyncio.sleep(0.3)
await _until_taken(taken, "red")
pipe.close("T4")
await asyncio.sleep(0.3)
taken.clear()
await _until_taken(taken, "yellow")
manifest = BrowserRecordingManifest.model_validate(await recordings.end(started["recording_id"]))
Comment thread
earakely-scale marked this conversation as resolved.
Resolved

assert [seen.tab for seen in manifest.video.followed] == ["T0", "T3", "T4", "T3"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
Recording,
record,
)
from agentenv_protocol.computer import annotate as annotate_module # noqa: E402
from agentenv_protocol.computer.annotate import ( # noqa: E402
COLOURS,
FADE_S,
Expand Down Expand Up @@ -318,14 +319,19 @@ def test_marks_no_frame_draws_are_kept_only_so_long():


@pytest.mark.asyncio
async def test_an_input_goes_ahead_when_its_mark_cant_be_made():
"""Reading the display for a mark fails: the scroll is still delivered, unmarked."""
async def test_an_input_goes_ahead_when_its_mark_cant_be_made(monkeypatch):
"""Making a scroll's mark fails: the scroll is still delivered, unmarked. The mark fails, not the display read it
makes, which the recording's own captures would race for."""
driver = Recorded()

def unmarkable(*args, **kwargs):
raise DeviceUnavailable("the mark couldn't be made")

async with http_client(driver) as http, mcp_session(driver) as session:
started = (await http.post(START, json={"request_id": "r", "fps": 10, "annotate": {}})).json()
recording = driver._computer.recordings.running[started["recording_id"]]
await session.call_tool("computer_screenshot", {})
driver.fail["display"] = DeviceUnavailable("the display went away")
monkeypatch.setattr(annotate_module, "input_marks", unmarkable)
scrolled = await session.call_tool("computer_scroll", {"direction": "down"})
marks = list(recording.annotations.marks)
await http.post(END, json={"recording_id": started["recording_id"]})
Expand Down
Loading