From 08e63ae9f642f78a40b273d52781cd9ea2f71a7e Mon Sep 17 00:00:00 2001 From: Paul Loeb Date: Fri, 13 Nov 2026 13:49:45 -0800 Subject: [PATCH] feat(shadow-requests): run judges on shadow responses A shadow variation's judges now score its response after it succeeds, with each judge's own sampling rate. Scores are tracked with the shadow's track data (isShadow, the shadow variationKey, the served runId), and ShadowResult.judge_results carries them. The shadow timeout still covers only the shadow handler, and skip_judges applies to the served variation. Judges run inside run_shadow's error handling, so run_shadow never raises. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/client/README.md | 4 +- ...ai_server.experimental.shadow_requests.txt | 3 +- .../experimental/shadow_requests.py | 74 ++++- .../src/launchdarkly_ai_server/judges.py | 36 +- .../src/launchdarkly_ai_server/lifecycle.py | 23 +- packages/client/tests/test_shadow_requests.py | 314 +++++++++++++++++- 6 files changed, 434 insertions(+), 20 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index edb5ff17..97fead47 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -391,7 +391,7 @@ elif result and task.evaluation_metric_key: > **Experimental.** Import shadow requests from `launchdarkly_ai_server.experimental.shadow_requests`. > These names may change in a minor release, so review the changelog when you upgrade. -A shadow request runs a candidate variation on real traffic without serving it. Use `shadow_config()` in place of `config()` at the call sites you want to shadow. It takes the same arguments, and `invoke()` and `stream()` return exactly what `config()` returns. When the context matches a targeting rule or the fallthrough that has shadows attached, each sampled shadow runs the same input on its own variation once the served variation succeeds. Its response is discarded. Its events are the normal `$ld:ai:*` events for that variation, with the served variation's `runId` and `isShadow: true`, so you can compare the two on the usual AI metrics. A shadow never delays or changes the served response, and its failures are only tracked. +A shadow request runs a candidate variation on real traffic without serving it. Use `shadow_config()` in place of `config()` at the call sites you want to shadow. It takes the same arguments, and `invoke()` and `stream()` return exactly what `config()` returns. When the context matches a targeting rule or the fallthrough that has shadows attached, each sampled shadow runs the same input on its own variation once the served variation succeeds. The shadow variation's judges score its response, which is then discarded. Its events are the normal `$ld:ai:*` events for that variation, with the served variation's `runId` and `isShadow: true`, so you can compare the two on the usual AI metrics. A shadow never delays or changes the served response, and its failures are only tracked. Shadows run in the background by default, each with a 30 second limit (`shadow_timeout_s`). `shutdown()` cancels any that are still running. Pass `on_task` to run them yourself instead: it receives a picklable `ShadowTask` for each sampled shadow, and `run_shadow(task, handlers)` runs it later and never raises. @@ -406,7 +406,7 @@ caller = shadow_config(key="my-ai-config", handler=handlers, on_task=queue.put_n shadow = await run_shadow(await queue.get(), handlers) ``` -A shadow runs as a plain `invoke()`, even when the served call streamed, with no tools and no judges. Pass handlers for every provider your shadow variations use, since a shadow with no matching handler is skipped. Plain `config()` never runs shadows. +A shadow runs as a plain `invoke()`, even when the served call streamed, with no tools. Its own judges run on its response after it succeeds, with their usual sampling rates, and each score is tracked under the shadow variation with `isShadow: true`. Each judge that fails is logged and skipped, and the others still score. Judges use the client already set up and never start one, so they are skipped after `shutdown()`. `ShadowResult.judge_results` holds the scores, and is present only when at least one judge scored. `skip_judges` applies to the served variation only. Pass handlers for every provider your shadow variations use, since a shadow with no matching handler is skipped. Plain `config()` never runs shadows. ### `graph(key, **options)` diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.shadow_requests.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.shadow_requests.txt index cfbe869b..a5d15c5b 100644 --- a/packages/client/api-report/launchdarkly_ai_server.experimental.shadow_requests.txt +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.shadow_requests.txt @@ -2,8 +2,9 @@ run_shadow(task: ShadowTask, handlers: list[ProviderHandler]) -> ShadowResult [async] shadow_config(*, key: str, handler: ProviderHandler | list[ProviderHandler] | None = None, tool_handlers: Mapping[str, Callable[..., Any] | NativeTool] | None = None, registry: Any = None, skip_judges: bool = False, shadow_timeout_s: float = 30.0, on_task: Callable[[ShadowTask], None] | None = None) -> ConfigInstance ShadowResult [class, @dataclass(frozen=True)] - __init__(usage: dict[str, int], track_data: dict[str, Any], error: str | None = None) + __init__(usage: dict[str, int], track_data: dict[str, Any], error: str | None = None, judge_results: dict[str, JudgeResult] | None = None) error: str | None = None + judge_results: dict[str, JudgeResult] | None = None track_data: dict[str, Any] usage: dict[str, int] ShadowTask [class, @dataclass(frozen=True)] diff --git a/packages/client/src/launchdarkly_ai_server/experimental/shadow_requests.py b/packages/client/src/launchdarkly_ai_server/experimental/shadow_requests.py index 5f59f108..30ea3447 100644 --- a/packages/client/src/launchdarkly_ai_server/experimental/shadow_requests.py +++ b/packages/client/src/launchdarkly_ai_server/experimental/shadow_requests.py @@ -5,8 +5,9 @@ LaunchDarkly attaches to the matched targeting rule or the fallthrough. The served variation runs exactly as ``config()`` runs it. Once it succeeds, each sampled shadow runs on the same input in the background, or is handed to ``on_task`` to run later with -``run_shadow()``. A shadow's response is discarded; it is tracked on the normal -``$ld:ai:*`` events with ``isShadow``. These names may change in a minor release, so +``run_shadow()``. The shadow variation's judges score its response, which is then +discarded; the shadow and its judge scores are tracked on the normal events with +``isShadow``. These names may change in a minor release, so review the changelog when you upgrade. """ @@ -25,15 +26,27 @@ from opentelemetry import context as otel_context -from ..client import ConfigInstance, _InternalConfigInstance +from ..client import ( + ConfigInstance, + _InternalConfigInstance, + _resolve_output_format_response, +) from ..conversation import _CONV_KEY, _conversation_id_from -from ..lifecycle import _fetch_variation, _parse_variation +from ..judge_scoring import judge_text +from ..judges import _run_judges as _run_core_judges +from ..lifecycle import ( + _existing_client, + _extract_existing_variation, + _fetch_variation, + _parse_variation, +) from ..registry import resolve_handlers from ..sdk_usage import report_usage from ..shutdown_hooks import register_shutdown_hook from ..tracking import execute_and_track, make_track_data from ..types import ( AiConfigRep, + JudgeResult, LDContext, NativeTool, ProviderHandler, @@ -52,15 +65,16 @@ _FALLTHROUGH = "fallthrough" _PENDING_SHADOWS = "__ld_pending_shadows" -_SHADOW_ONLY_KEYS = ("tools", "judgeConfiguration", "shadowConfiguration") +_SHADOW_ONLY_KEYS = ("tools", "shadowConfiguration") _BUCKET_SCALE = float(0xFFFFFFFFFFFFFFF) @dataclass(frozen=True) class ShadowTask: """ - Everything needed to run and track one sampled shadow, with no flag evaluation. - Plain data, so it survives ``pickle``. Run it with ``run_shadow(task, handlers)``. + Everything needed to run and track one sampled shadow. Running it evaluates no + flag except the shadow's judges. Plain data, so it survives ``pickle``. Run it + with ``run_shadow(task, handlers)``. """ config_key: str @@ -79,11 +93,16 @@ class ShadowTask: @dataclass(frozen=True) class ShadowResult: - """The outcome of one shadow run. There is no response text: it is discarded.""" + """ + The outcome of one shadow run. There is no response text: it is discarded. + ``judge_results`` holds the shadow variation's judge scores, present only when + at least one judge scored. + """ usage: dict[str, int] track_data: dict[str, Any] error: str | None = None + judge_results: dict[str, JudgeResult] | None = None _running: set[asyncio.Task[ShadowResult]] = set() @@ -430,11 +449,48 @@ async def _run(task: ShadowTask, handlers: list[ProviderHandler]) -> ShadowResul span_attributes=task.span_attributes, feature_flag_event=False, ) + judge_results = await _run_judges(task, handler, handlers, result) except _ShadowTimeout: return _failed(task, "timeout") except Exception as err: return _failed(task, str(err) or type(err).__name__) - return ShadowResult(usage=result["usage"], track_data=result["track_data"]) + return ShadowResult( + usage=result["usage"], + track_data=result["track_data"], + judge_results=judge_results or None, + ) + + +async def _run_judges( + task: ShadowTask, + handler: ProviderHandler, + handlers: list[ProviderHandler], + result: dict[str, Any], +) -> dict[str, JudgeResult]: + """Scores the shadow's response with its own judges, outside the shadow timeout. + + Judges resolve through the client already set up and never start one, so + with no live client they are skipped.""" + if _existing_client() is None: + return {} + try: + response = _resolve_output_format_response( + result["response"], task.shadow_config.get("outputFormat") + ) + return await _run_core_judges( + config=task.shadow_config, + user_context=task.user_context, + handler=handler, + handlers=handlers, + user_input=task.user_input, + trajectory=result.get("trajectory", ""), + llm_response=judge_text(response), + base_track_data=result["track_data"], + resolve=_extract_existing_variation, + ) + except Exception: + logger.warning("Shadow request judges failed", exc_info=True) + return {} @contextmanager diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index af17c697..c63a4e5b 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -2,7 +2,7 @@ import logging import random -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from contextlib import contextmanager from typing import Any @@ -113,6 +113,38 @@ async def run_judges( unchanged, and so is a judge for a config with no tools. """ from .lifecycle import extract_variation + + return await _run_judges( + config=config, + user_context=user_context, + handler=handler, + handlers=handlers, + user_input=user_input, + llm_response=llm_response, + base_track_data=base_track_data, + tool_handlers=tool_handlers, + graph_key=graph_key, + trajectory=trajectory, + resolve=extract_variation, + ) + + +async def _run_judges( + *, + config: AiConfigRep, + user_context: LDContext, + handler: ProviderHandler, + handlers: list[ProviderHandler] | None, + user_input: str | None, + llm_response: str, + base_track_data: TrackData, + tool_handlers: Mapping[str, Callable[..., Any] | NativeTool] | None = None, + graph_key: str | None = None, + trajectory: str = "", + resolve: Callable[[str, LDContext], Awaitable[dict[str, Any]]], +) -> dict[str, JudgeResult]: + """:func:`run_judges`, resolving each judge's variation with ``resolve``. + A judge whose resolution raises is logged and skipped.""" from .tracking import execute_and_track judge_results: dict[str, JudgeResult] = {} @@ -134,7 +166,7 @@ async def run_judges( judge_key = judge["key"] try: - variation = await extract_variation(judge_key, user_context) + variation = await resolve(judge_key, user_context) judge_ai_config: AiConfigRep = variation["config"] judge_meta = variation["meta"] diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 856917b4..f7450fda 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -130,6 +130,11 @@ def note_ai_invocation() -> None: _ai_invoked = True +def _existing_client() -> LDClientInterface | None: + """Returns the client already set up, or ``None``. Never starts one.""" + return _client + + def get_client() -> LDClientInterface: """ Returns the singleton LaunchDarkly client. @@ -764,14 +769,28 @@ async def extract_variation( return _parse_variation(config_key, raw) +async def _extract_existing_variation( + config_key: str, context: dict[str, Any] +) -> dict[str, Any]: + """Like :func:`extract_variation`, but uses the client already set up and + never starts one, raising when there is none.""" + raw, _ = await _fetch_variation(config_key, context, existing_client_only=True) + return _parse_variation(config_key, raw) + + async def _fetch_variation( - config_key: str, context: dict[str, Any], *, with_reason: bool = False + config_key: str, + context: dict[str, Any], + *, + with_reason: bool = False, + existing_client_only: bool = False, ) -> tuple[Any, Any]: """The raw variation value, and its evaluation reason when ``with_reason`` is set and the client has ``variation_detail``. Otherwise the reason is ``None``.""" from .utils import to_ld_context # late import avoids circular dependency - await init_client() + if not existing_client_only: + await init_client() client = get_client() # The real ldclient.variation() expects an ldclient.Context, not a plain dict. # Convert when running against the real SDK; test mocks accept dicts directly. diff --git a/packages/client/tests/test_shadow_requests.py b/packages/client/tests/test_shadow_requests.py index 313b8278..89597d40 100644 --- a/packages/client/tests/test_shadow_requests.py +++ b/packages/client/tests/test_shadow_requests.py @@ -64,8 +64,10 @@ def _entry(**overrides: Any) -> dict[str, Any]: return entry -def _variation_b(enabled: bool = True) -> dict[str, Any]: - return { +def _variation_b( + enabled: bool = True, judges: list[dict[str, Any]] | None = None +) -> dict[str, Any]: + variation: dict[str, Any] = { "_ldMeta": { "enabled": enabled, "variationKey": "b", @@ -78,8 +80,10 @@ def _variation_b(enabled: bool = True) -> dict[str, Any]: "provider": {"name": "ShadowProvider"}, "instructions": "Shadow.", "tools": {"search": {"name": "search", "type": "function", "parameters": {}}}, - "judgeConfiguration": {"judges": [{"key": "j", "samplingRate": 1}]}, } + if judges is not None: + variation["judgeConfiguration"] = {"judges": judges} + return variation def _variation_a( @@ -645,7 +649,6 @@ async def test_same_input_on_the_stripped_config(self) -> None: assert call["variables"]["topic"] == "x" assert call["config"]["instructions"] == "Shadow." assert "tools" not in call["config"] - assert "judgeConfiguration" not in call["config"] assert "_ldMeta" not in call["config"] assert not call["tools"] @@ -1073,3 +1076,306 @@ async def test_shadow_with_malformed_tools_still_runs(self) -> None: _, _, b = await _invoke(_make_client(_variation_a(variations={"b": variation}))) assert len(b.calls) == 1 assert "tools" not in b.calls[0]["config"] + + +JUDGE_VERDICT = '{"score": 0.8, "reasoning": "good"}' + + +def _judge_variation() -> dict[str, Any]: + return { + "_ldMeta": { + "enabled": True, + "variationKey": "jv", + "version": 1, + "mode": "judge", + }, + "model": {"name": "judge-model"}, + "provider": {"name": "JudgeProvider"}, + "instructions": "Judge.", + "evaluationMetricKey": "judge-score", + } + + +def _judged_client( + b_rate: float = 1, a_judged: bool = False +) -> tuple[MagicMock, dict[str, Any]]: + value = _variation_a( + variations={"b": _variation_b(judges=[{"key": "j", "samplingRate": b_rate}])} + ) + if a_judged: + value["judgeConfiguration"] = {"judges": [{"key": "j", "samplingRate": 1}]} + client = _make_client(value) + + async def variation(key: str, *args: Any, **kwargs: Any) -> dict[str, Any]: + return _judge_variation() if key == "j" else value + + client.variation = AsyncMock(side_effect=variation) + return client, value + + +def _judge_events(client: MagicMock) -> list[dict[str, Any]]: + return [d for n, d in _events(client) if n == "judge-score"] + + +async def _invoke_judged( + client: MagicMock, + *, + b: _Recorder | None = None, + judge: _Recorder | None = None, + **shadow_args: Any, +) -> tuple[Any, _Recorder, _Recorder, _Recorder]: + _use(client) + a = _Recorder("TestProvider", "real answer") + b = b or _Recorder("ShadowProvider", "shadow answer") + judge = judge or _Recorder("JudgeProvider", JUDGE_VERDICT) + response = await shadow_config( + key="flag", handler=[a.handler, b.handler, judge.handler], **shadow_args + ).invoke("question", CONTEXT) + await sr._wait_for_shadows_for_testing() + return response, a, b, judge + + +class TestJudges: + async def test_judge_scores_the_shadow_response(self) -> None: + client, _ = _judged_client() + response, _, b, judge = await _invoke_judged(client) + assert response.response == "real answer" + assert len(b.calls) == 1 + assert len(judge.calls) == 1 + assert judge.calls[0]["variables"]["response_to_evaluate"] == "shadow answer" + assert judge.calls[0]["variables"]["input"] == "question" + + async def test_judge_runs_after_the_caller_returns(self) -> None: + client, _ = _judged_client() + _use(client) + a = _Recorder("TestProvider", "real answer") + b = _Recorder("ShadowProvider", "shadow answer") + judge = _Recorder("JudgeProvider", JUDGE_VERDICT) + await shadow_config( + key="flag", handler=[a.handler, b.handler, judge.handler] + ).invoke("question", CONTEXT) + assert judge.calls == [] + await sr._wait_for_shadows_for_testing() + assert len(judge.calls) == 1 + + async def test_shadow_judge_event_is_tracked_under_the_shadow(self) -> None: + client, _ = _judged_client(a_judged=True) + await _invoke_judged(client) + shadow = [d for d in _judge_events(client) if d.get("isShadow")] + real = [d for d in _judge_events(client) if not d.get("isShadow")] + assert len(shadow) == 1 + assert len(real) == 1 + assert shadow[0]["variationKey"] == "b" + assert shadow[0]["shadowRequestId"] == "shadow-1" + assert shadow[0]["primaryVariationKey"] == "a" + assert shadow[0]["judgeConfigKey"] == "j" + assert shadow[0]["ownerKind"] == "member" + assert shadow[0]["ownerId"] == "member-1" + assert shadow[0]["runId"] == real[0]["runId"] + assert not SHADOW_FIELDS & real[0].keys() + + async def test_shadow_handler_receives_the_judge_configuration(self) -> None: + client, _ = _judged_client() + _, _, b, _ = await _invoke_judged(client) + assert b.calls[0]["config"]["judgeConfiguration"] == { + "judges": [{"key": "j", "samplingRate": 1}] + } + + async def test_no_judge_without_a_matching_shadow_handler(self) -> None: + client, _ = _judged_client() + _use(client) + a = _Recorder("TestProvider", "real answer") + judge = _Recorder("JudgeProvider", JUDGE_VERDICT) + response = await shadow_config( + key="flag", handler=[a.handler, judge.handler] + ).invoke("question", CONTEXT) + await sr._wait_for_shadows_for_testing() + assert response.response == "real answer" + assert judge.calls == [] + assert _judge_events(client) == [] + assert _shadow_events(client) == [] + + async def test_judge_sees_the_tasks_conversation_id(self) -> None: + client, _ = _judged_client() + with conversation_id("conv-1"): + task = await _deferred_task_judged(client) + b = _Recorder("ShadowProvider", "shadow answer") + judge = _Recorder("JudgeProvider", JUDGE_VERDICT) + with conversation_id("worker-conversation"): + await run_shadow(task, [b.handler, judge.handler]) + assert b.calls[0]["conversation_id"] == "conv-1" + assert judge.calls[0]["conversation_id"] == "conv-1" + + async def test_no_judge_without_a_live_client( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + client, _ = _judged_client() + task = await _deferred_task_judged(client) + init = AsyncMock() + monkeypatch.setattr(lifecycle_module, "init_client", init) + monkeypatch.setattr(sr, "_existing_client", lambda: None) + judge = _Recorder("JudgeProvider", JUDGE_VERDICT) + result = await run_shadow( + task, [_Recorder("ShadowProvider", "shadow answer").handler, judge.handler] + ) + assert result.error is None + assert judge.calls == [] + assert result.judge_results is None + init.assert_not_awaited() + + async def test_shadow_judges_never_start_a_client( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + init = AsyncMock() + monkeypatch.setattr(lifecycle_module, "init_client", init) + lifecycle_module._set_client_for_testing(None) + with pytest.raises(RuntimeError): + await lifecycle_module._extract_existing_variation("j", CONTEXT) + init.assert_not_awaited() + + @pytest.mark.parametrize(("rate", "runs"), [(0, 0), (1, 1)]) + async def test_judge_sampling_rate_applies(self, rate: float, runs: int) -> None: + client, _ = _judged_client(b_rate=rate) + _, _, b, judge = await _invoke_judged(client) + assert len(b.calls) == 1 + assert len(judge.calls) == runs + assert len(_judge_events(client)) == runs + + @pytest.mark.parametrize( + "b", + [ + _Recorder("ShadowProvider", error=RuntimeError("boom")), + _Recorder("ShadowProvider", sleep_s=1), + ], + ids=["error", "timeout"], + ) + async def test_no_judge_when_the_shadow_fails(self, b: _Recorder) -> None: + client, _ = _judged_client() + _, _, _, judge = await _invoke_judged(client, b=b, shadow_timeout_s=0.05) + assert judge.calls == [] + assert _judge_events(client) == [] + + async def test_judge_failure_is_not_a_shadow_failure( + self, caplog: pytest.LogCaptureFixture + ) -> None: + client, _ = _judged_client() + task = await _deferred_task_judged(client) + judge = _Recorder("JudgeProvider", "not json") + with caplog.at_level(logging.ERROR): + result = await run_shadow( + task, + [_Recorder("ShadowProvider", "shadow answer").handler, judge.handler], + ) + assert len(judge.calls) == 1 + assert result.error is None + assert result.judge_results is None + assert "$ld:ai:generation:success" in [e[0] for e in _shadow_events(client)] + assert any("Judge 'j' failed" in m for m in caplog.messages) + + async def test_one_judge_failing_leaves_the_others_scoring( + self, caplog: pytest.LogCaptureFixture + ) -> None: + value = _variation_a( + variations={ + "b": _variation_b( + judges=[ + {"key": "failing", "samplingRate": 1}, + {"key": "j", "samplingRate": 1}, + ] + ) + } + ) + client = _make_client(value) + failing = {**_judge_variation(), "provider": {"name": "FailingProvider"}} + + async def variation(key: str, *args: Any, **kwargs: Any) -> dict[str, Any]: + if key == "failing": + return failing + return _judge_variation() if key == "j" else value + + client.variation = AsyncMock(side_effect=variation) + task = await _deferred_task_judged(client) + broken = _Recorder("FailingProvider", error=RuntimeError("provider down")) + judge = _Recorder("JudgeProvider", JUDGE_VERDICT) + with caplog.at_level(logging.ERROR): + result = await run_shadow( + task, + [ + _Recorder("ShadowProvider", "shadow answer").handler, + broken.handler, + judge.handler, + ], + ) + assert len(broken.calls) == 1 + assert len(judge.calls) == 1 + assert result.error is None + assert result.judge_results is not None + assert set(result.judge_results) == {"j"} + assert len(_judge_events(client)) == 1 + assert any("Judge 'failing' failed" in m for m in caplog.messages) + + async def test_timeout_does_not_cover_judges(self) -> None: + client, _ = _judged_client() + task = dataclasses.replace(await _deferred_task_judged(client), timeout_s=0.2) + judge = _Recorder("JudgeProvider", JUDGE_VERDICT, sleep_s=0.3) + result = await run_shadow( + task, [_Recorder("ShadowProvider", "shadow answer").handler, judge.handler] + ) + assert result.error is None + assert result.judge_results is not None + assert result.judge_results["j"].score == 0.8 + + async def test_result_carries_judge_results_only_when_a_judge_ran(self) -> None: + client, _ = _judged_client() + task = await _deferred_task_judged(client) + handlers = [ + _Recorder("ShadowProvider", "shadow answer").handler, + _Recorder("JudgeProvider", JUDGE_VERDICT).handler, + ] + judged = await run_shadow(task, handlers) + assert judged.judge_results is not None + assert judged.judge_results["j"].score == 0.8 + assert "shadow answer" not in repr(judged) + + unjudged = await run_shadow( + await _deferred_task(_make_client(_variation_a())), handlers + ) + assert unjudged.judge_results is None + + async def test_skip_judges_applies_to_the_served_variation_only(self) -> None: + client, _ = _judged_client(a_judged=True) + response, _, _, judge = await _invoke_judged(client, skip_judges=True) + assert response.judge_tasks is not None + assert len(judge.calls) == 1 + assert [d.get("isShadow") for d in _judge_events(client)] == [True] + + async def test_deferred_and_background_emit_the_same_judge_events(self) -> None: + background, _ = _judged_client() + await _invoke_judged(background) + + deferred, _ = _judged_client() + task = await _deferred_task_judged(deferred) + deferred.track.reset_mock() + await run_shadow( + task, + [ + _Recorder("ShadowProvider", "shadow answer").handler, + _Recorder("JudgeProvider", JUDGE_VERDICT).handler, + ], + ) + assert _without_run_id(_shadow_events(background)) == _without_run_id( + _shadow_events(deferred) + ) + + async def test_judged_task_survives_pickle(self) -> None: + client, _ = _judged_client() + task = await _deferred_task_judged(client) + assert pickle.loads(pickle.dumps(task)) == task + assert task.shadow_config["judgeConfiguration"]["judges"][0]["key"] == "j" + + +async def _deferred_task_judged(client: MagicMock) -> ShadowTask: + tasks: list[ShadowTask] = [] + await _invoke_judged(client, on_task=tasks.append) + assert len(tasks) == 1 + return tasks[0]