From 939e543d1ddc101a5783e024b1258cfe6addfa5e Mon Sep 17 00:00:00 2001 From: duongthimy Date: Mon, 16 Nov 2026 18:31:02 +0100 Subject: [PATCH 1/3] feat(gooddata-eval): evaluate Analytics Catalog title/description Add the single-shot kind catalog_metadata for the Analytics Catalog generateTitle / generateDescription endpoints. - catalog_input on dataset items: object type, object id, output - client asks again once on a 200 answer with no usable text and records unusable_retries - evaluator: format checks (generated, no note, word / sentence limits, no markup, no ids) plus judge criteria graded like dashboard_summary; expect_note items pass on no text + a note - violation judge decides presence only: the item question carries the object's real metadata, which made the judge grade correctness and score a stated wrong fact as absent - evaluate_catalog_metadata: K-run, gate and Langfuse entry point for the Tavern shim - make calibrate-catalog-metadata with a calibration fixture - the takeaway and catalog clients share the retry predicate and is_transient in core/_retry.py jira: QA-30180 risk: low --- packages/gooddata-eval/AGENTS.md | 15 +- packages/gooddata-eval/Makefile | 4 + packages/gooddata-eval/README.md | 53 +++- .../calibrate_catalog_metadata_judge.py | 62 ++++ .../src/gooddata_eval/cli/main.py | 7 +- .../src/gooddata_eval/core/_retry.py | 27 ++ .../gooddata_eval/core/catalog/__init__.py | 1 + .../gooddata_eval/core/catalog/evaluate.py | 268 ++++++++++++++++ .../gooddata_eval/core/catalog/http_client.py | 97 ++++++ .../core/dataset/langfuse_source.py | 35 ++- .../gooddata_eval/core/evaluators/__init__.py | 6 +- .../core/evaluators/catalog_metadata.py | 162 ++++++++++ .../src/gooddata_eval/core/models.py | 19 +- .../core/takeaway/http_client.py | 29 +- .../catalog_metadata_calibration.json | 290 ++++++++++++++++++ .../tests/test_catalog_metadata_client.py | 177 +++++++++++ .../tests/test_catalog_metadata_evaluate.py | 256 ++++++++++++++++ .../tests/test_catalog_metadata_evaluator.py | 125 ++++++++ packages/gooddata-eval/tests/test_cli.py | 19 ++ .../tests/test_langfuse_source.py | 52 ++++ packages/gooddata-eval/tests/test_models.py | 8 + packages/gooddata-eval/tests/test_runner.py | 1 + 22 files changed, 1670 insertions(+), 43 deletions(-) create mode 100644 packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/catalog/__init__.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/catalog/evaluate.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py create mode 100644 packages/gooddata-eval/tests/fixtures/catalog_metadata_calibration.json create mode 100644 packages/gooddata-eval/tests/test_catalog_metadata_client.py create mode 100644 packages/gooddata-eval/tests/test_catalog_metadata_evaluate.py create mode 100644 packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 4b33b57a2..f7c298e0d 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -28,6 +28,7 @@ experiment. The newest and most actively developed package in the repo. | `core/chat/` | SSE client for the agent's streaming chat endpoint | | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting | | `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls. Also used by the agentic document skill to write the takeaways the copilot leaves to the editor, before the narrative is graded | +| `core/catalog/` | HTTP client for the Analytics Catalog `generateTitle` / `generateDescription` endpoints — single-shot like `core/takeaway/`, asking again once on a 200 answer without usable text — plus `evaluate_catalog_metadata`, the K-run, gate and Langfuse entry point the Tavern shim calls | | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights | | `core/evaluators/` | single-shot evaluators and their registry | | `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) | @@ -62,7 +63,7 @@ implementation, sometimes under different `test_kind` strings (`search_tool` vs its own shape. `test_kind` on the item is what labels the result, not the evaluator class, which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim. `dashboard_summary` items additionally need `summary_input`, `document_takeaway` items -`takeaway_input`. +`takeaway_input`, `catalog_metadata` items `catalog_input`. ## Running the pipeline @@ -158,8 +159,8 @@ Run this before guessing a `--model` string. | `GD_EVAL_JUDGE_MODEL` | judge model, same as `--judge-model` | | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` | | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` | -| `GOODDATA_EVAL_CHAT_*` | retry and backoff knobs shared by the chat and takeaway clients (`core/_retry.py`), and the SSE turn and item timeouts | -| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary and document-takeaway runs are not labelled | +| `GOODDATA_EVAL_CHAT_*` | retry and backoff knobs shared by the chat, takeaway and catalog clients (`core/_retry.py`), and the SSE turn and item timeouts | +| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary, document-takeaway and catalog-metadata runs are not labelled | | `GOODDATA_EVAL_JOIN_GENAI_TRACE` | on unless set to `0`, `false`, `no`, `off` or empty. An agentic item's chat turns ask gen-ai to open them under gd-eval's experiment root, so each item is one Langfuse trace with its scores written once. A gen-ai that drops caller baggage keeps today's separate traces; see the gotcha below | | `GD_EVAL_TIMERS` | same as `--timers` | @@ -185,14 +186,14 @@ Adding a kind to `PARALLEL_SAFE_TEST_KINDS` is a deliberate assertion that it is which nothing in the package can prove for you. **The SSE client's retry predicate is load-bearing.** The backoff loop and the retryable -status set (429/500/502/503/504) live in `core/_retry.py`, shared by the chat and takeaway -clients and tuned by the same `GOODDATA_EVAL_CHAT_*` knobs. `core/chat/sse_client.py` +status set (429/500/502/503/504) live in `core/_retry.py`, shared by the chat, takeaway and +catalog clients and tuned by the same `GOODDATA_EVAL_CHAT_*` knobs. `core/chat/sse_client.py` retries that status set and `httpx.RemoteProtocolError` (a mid-stream disconnect), and treats a `METADATA_SYNC_IN_PROGRESS` payload as transient. The `RemoteProtocolError` case was added after it was confirmed live to contaminate a small percentage of visualization runs with a hard fail and no retry. Narrowing that predicate reintroduces the problem. The -takeaway client retries the same status set plus connection failures, but not a read -timeout: a slow page would spend the whole timeout again on every attempt. +takeaway and catalog clients retry the same status set plus connection failures, but not a +read timeout: a slow page would spend the whole timeout again on every attempt. **Langfuse trace linking is deliberately off the item critical path.** Polling for trace ingestion has no pass/fail signal and inflates or misattributes per-item latency, so diff --git a/packages/gooddata-eval/Makefile b/packages/gooddata-eval/Makefile index 59c2f679e..4bf22f465 100644 --- a/packages/gooddata-eval/Makefile +++ b/packages/gooddata-eval/Makefile @@ -4,3 +4,7 @@ include ../../project_common.mk .PHONY: calibrate-takeaway calibrate-takeaway: uv run --package gooddata-eval --extra llm-judge python scripts/calibrate_takeaway_judge.py + +.PHONY: calibrate-catalog-metadata +calibrate-catalog-metadata: + uv run --package gooddata-eval --extra llm-judge python scripts/calibrate_catalog_metadata_judge.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 6ee40dc5b..9749259d1 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -132,8 +132,8 @@ gd-eval run \ | `--runs K` | `2` | Independent runs per item. | | `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Kinds that repeat K runs only — `power` is refused when the dataset also has non-agentic items or `agentic_conversation`, which are always decided on pass@K. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | -| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary` and `document_takeaway`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | -| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` and `document_takeaway` items go through their own endpoints, which have no such option. | +| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway` and `catalog_metadata`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | +| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary`, `document_takeaway` and `catalog_metadata` items go through their own endpoints, which have no such option. | **Concurrency and workspace safety.** Agentic kinds that create workspace objects (`agentic_metric_skill`, `agentic_alert_skill`, `agentic_conversation`, `agentic_kda_skill`) always run one at a @@ -708,7 +708,8 @@ A dataset is a folder of `.json` files, one per question: ``` Supported `test_kind` values: `visualization`, `metric_skill`, `alert_skill`, -`search_tool`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway`. +`search_tool`, `general_question`, `guardrail`, `dashboard_summary`, `document_takeaway`, +`catalog_metadata`. ### `visualization` items @@ -866,6 +867,51 @@ and raises only when no run completed. A read timeout is not retried, and the CI point records it as an errored run too. Any other 4xx and an unsupported URL scheme are never retried and fail the item in both. +### `catalog_metadata` items + +Catalog items call the Analytics Catalog generate endpoints +(`POST /api/v1/actions/workspaces/{ws}/ai/analyticsCatalog/generateTitle` or +`/generateDescription`) for one object, named by a `catalog_input` block. In a Langfuse +dataset the block goes in the item's `metadata` or its `input` object (`input` wins); a +value that is not an object fails the item. An item carrying +`catalog_input` is a `catalog_metadata` item unless it declares another `test_kind`. The +judge reads the item's `question` as the ground truth, so state the object's real metadata +there in plain words (what a metric computes, which filters a visualization has). + +```json +{ + "id": "catalog-001", + "dataset_name": "analytics_catalog_generate", + "test_kind": "catalog_metadata", + "question": "Description for the metric '# of Single Orders'. Its MAQL counts customers who placed exactly one order.", + "catalog_input": {"object_type": "Metric", "object_id": "of_orders_1", "output": "description"}, + "expected_output": { + "must_include": ["Says that it counts customers who placed exactly one order."], + "must_not_include": ["Says the metric counts orders rather than customers."], + "rubric": [] + } +} +``` + +`object_type` is one of `Visualization`, `Dashboard`, `Metric`, `Fact`, `Attribute`; +`output` is `title` or `description`; unknown keys are rejected. Every `expected_output` +key is optional: + +- `expect_note` — `true` for an object the endpoint must decline: the item passes on no + text plus a non-empty `note`, and nothing else is graded. +- `format` — overrides of the deterministic checks, which gate pass/fail: `max_words` + (title 10, description 80), `max_sentences` (title 1, description 4), `no_end_period` + (title), `no_markup`, `no_ids`. `false` or `null` switches a check off; unknown keys are + rejected. A generated text (`generated`) and an empty `note` (`no_note`) always apply. +- `must_include`, `must_not_include`, `rubric` — judge criteria, graded like + `dashboard_summary`. Criteria may name the object's metadata; the judge sees no data. + +A 200 answer with no usable text (a null value without a note, or the placeholder +`"string"`) is asked again once; `unusable_retries` in the item detail records it, with +`word_count`. The judge steps are calibrated with `make calibrate-catalog-metadata` (needs +`OPENAI_API_KEY`); re-run it after changing them. Request errors are handled as for +`document_takeaway`; the CI entry point is `evaluate_catalog_metadata`. + ### Items asked in a dashboard context (`user_context`) An item can be asked the way a user asks from an open dashboard or an attached widget. @@ -966,6 +1012,7 @@ from that turn on until another turn changes it, as the attached context does in | `guardrail` | Refusal/redirect (visualization response auto-fails) | `[llm-judge]` | | `dashboard_summary` | Dashboard summary (via `/summary` endpoint) scored against a rubric by LLM | `[llm-judge]` | | `document_takeaway` | One short takeaway for a page (via `/takeaway` endpoint): deterministic format checks plus a rubric scored by LLM | `[llm-judge]` | +| `catalog_metadata` | Analytics Catalog title or description for one object (via the `analyticsCatalog/generate*` endpoints): format checks plus a rubric scored by LLM | `[llm-judge]` | ## Optional extras diff --git a/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py b/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py new file mode 100644 index 000000000..7d96b7255 --- /dev/null +++ b/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py @@ -0,0 +1,62 @@ +# (C) 2026 GoodData Corporation +"""Does the real judge grade catalog_metadata criteria on the right side? + +tests/test_catalog_metadata_evaluator.py pins the wiring; it cannot pin the verdict, because +the verdict is an LLM's. This does, against tests/fixtures/catalog_metadata_calibration.json: +each case is one item question (the object's real metadata), one criterion and one text, and +the criterion must be satisfied ("pass") or not ("fail") on every run. + +The fixture keeps texts that state a wrong fact the question contradicts: a violation judge +that grades correctness instead of presence scores those as absent and lets them pass. + +Run it after changing _POSITIVE_STEPS / _VIOLATION_STEPS: + + OPENAI_API_KEY=... make calibrate-catalog-metadata + +It is a script rather than a test because CI has no OPENAI_API_KEY. Exits 1 on any mismatch. +""" + +import json +import os +import sys +from pathlib import Path + +from gooddata_eval.core.evaluators.catalog_metadata import CatalogMetadataEvaluator +from gooddata_eval.core.models import CatalogInput, ChatResult, DatasetItem + +RUNS = 3 +FIXTURE = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "catalog_metadata_calibration.json" +_FIELDS = {"include": "must_include", "exclude": "must_not_include"} + + +def main() -> int: + if not os.environ.get("OPENAI_API_KEY"): + raise SystemExit("OPENAI_API_KEY required") + cases = json.loads(FIXTURE.read_text()) + evaluator = CatalogMetadataEvaluator() + print(f"judge model: {evaluator._positive_judge.model}, runs: {RUNS}") + mismatches = 0 + for case in cases: + item = DatasetItem( + id=case["label"], + dataset_name="catalog_metadata_calibration", + test_kind="catalog_metadata", + question=case["question"], + expected_output={_FIELDS[case["kind"]]: [case["criterion"]], "format": {}}, + catalog_input=CatalogInput.model_validate( + {"object_type": "Metric", "object_id": "calibration", "output": case["output"]} + ), + ) + chat = ChatResult(text_response=case["text"], raw_response={"note": None}) + key = f"{case['kind']}_0" + satisfied = sum(evaluator.evaluate(item, chat).detail.get(key) is True for _ in range(RUNS)) + want = RUNS if case["want"] == "pass" else 0 + ok = satisfied == want + mismatches += not ok + print(f"{case['label']:48} want {case['want']:4} satisfied {satisfied}/{RUNS} {'OK' if ok else 'MISMATCH'}") + print(f"\n{'all calibration texts on the right side' if not mismatches else f'{mismatches} mismatch(es)'}") + return 1 if mismatches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index efbbf1069..071e455f9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -15,6 +15,7 @@ from rich.table import Table from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items +from gooddata_eval.core.catalog.http_client import CatalogMetadataClient from gooddata_eval.core.chat.sse_client import ChatClient, set_default_item_timeout, set_default_turn_timeout from gooddata_eval.core.config import ( DEFAULT_GATE, @@ -48,7 +49,8 @@ class _RoutingBackend: """Dispatch each item to the right backend by test_kind. Kinds in `by_kind` go to their dedicated endpoint client (`dashboard_summary` to the - summary endpoint, `document_takeaway` to the takeaway endpoint); everything else uses + summary endpoint, `document_takeaway` to the takeaway endpoint, `catalog_metadata` to the + Analytics Catalog generate endpoints); everything else uses the conversational chat endpoint. """ @@ -620,6 +622,9 @@ def on_langfuse_item_done( "document_takeaway": TakeawayClient( host=config.host, token=config.token, workspace_id=config.workspace_id ), + "catalog_metadata": CatalogMetadataClient( + host=config.host, token=config.token, workspace_id=config.workspace_id + ), }, ) try: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/_retry.py b/packages/gooddata-eval/src/gooddata_eval/core/_retry.py index 982f8cb74..8edf71f69 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/_retry.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/_retry.py @@ -13,6 +13,8 @@ from collections.abc import Callable from typing import TypeVar +import httpx + _log = logging.getLogger(__name__) T = TypeVar("T") @@ -24,6 +26,31 @@ RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504}) +# Failures of the single-shot endpoint clients (takeaway, catalog) that say nothing about the +# request. A read timeout is left out: the server was computing, and retrying spends the whole +# timeout again on every attempt. +RETRYABLE_TRANSPORT_ERRORS = (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError, httpx.PoolTimeout) + + +def is_retryable_request_error(exc: Exception) -> bool: + """Whether a single-shot endpoint client retries this failure: a retryable status or transport error.""" + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code in RETRYABLE_STATUS_CODES + return isinstance(exc, RETRYABLE_TRANSPORT_ERRORS) + + +def is_transient(exc: httpx.HTTPError) -> bool: + """Whether a failure left after the client's retries says nothing about the request. + + 429, 5xx and transport failures (a read timeout included) are transient; any other 4xx + (flag off, no agent, invalid body) and an unsupported URL scheme are setup problems. + """ + if isinstance(exc, httpx.HTTPStatusError): + status = exc.response.status_code + return status == 429 or status >= 500 + return isinstance(exc, httpx.TransportError) and not isinstance(exc, httpx.UnsupportedProtocol) + + def int_env(name: str, default: int) -> int: """Read an int from the environment, falling back to ``default`` when unset or blank.""" raw = os.getenv(name) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/catalog/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/catalog/__init__.py new file mode 100644 index 000000000..efe7c60c8 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/catalog/__init__.py @@ -0,0 +1 @@ +# (C) 2026 GoodData Corporation diff --git a/packages/gooddata-eval/src/gooddata_eval/core/catalog/evaluate.py b/packages/gooddata-eval/src/gooddata_eval/core/catalog/evaluate.py new file mode 100644 index 000000000..db1732f2e --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/catalog/evaluate.py @@ -0,0 +1,268 @@ +# (C) 2026 GoodData Corporation +"""Entry point for evaluating one catalog_metadata item K times, for the Tavern shim. + +The CLI evaluates catalog items through the single-shot runner; CI goes through +``evaluate_catalog_metadata``, which owns the K-run loop, the gate and the Langfuse +experiment item the same way ``evaluate_document_takeaway`` does, so the nightly report +reads it unchanged. + +The generate endpoints return no conversation id, so there is no gen-ai trace to link: +every score is written to gd-eval's own experiment span. + +A run whose request still fails transiently after the client's retries (5xx, 429, a +connection failure or a read timeout) is recorded as an errored run and left ungraded; +the item raises it only when no run completed. Any other 4xx (no AI entitlement, invalid +body) or an unsupported URL scheme is a setup problem and raises on the first run. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any + +import httpx + +from gooddata_eval.core._retry import is_transient +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.catalog.http_client import CatalogMetadataClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.catalog_metadata import CatalogMetadataEvaluator +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ChatResult, DatasetItem +from gooddata_eval.core.timing import PhaseTimings, sum_timings + +_TEST_KIND = "catalog_metadata" +_COUNT_SCORES = ("word_count", "unusable_retries") + + +class CatalogMetadataAssertionError(AgenticAssertionError): + """Raised when a catalog_metadata evaluation fails its gate.""" + + +@dataclass +class _CatalogRun: + chat_result: ChatResult + evaluation: ItemEvaluation + latency_s: float + timings: PhaseTimings + request_error: httpx.HTTPError | None = None + + @property + def judge_error(self) -> str | None: + return self.evaluation.error + + @property + def checks(self) -> dict[str, bool]: + return {k: v for k, v in self.evaluation.detail.items() if isinstance(v, bool)} + + +def _dataset_item(dataset_item: dict, dataset_name: str) -> DatasetItem: + item = DatasetItem.model_validate( + {"id": "", "test_kind": _TEST_KIND, "expected_output": {}, **dataset_item, "dataset_name": dataset_name} + ) + if item.catalog_input is None: + raise ValueError(f"catalog_metadata item '{item.id}' is missing required 'catalog_input'.") + return item + + +def _run_once(client: CatalogMetadataClient, evaluator: CatalogMetadataEvaluator, item: DatasetItem) -> _CatalogRun: + started = time.monotonic() + try: + chat_result = client.ask(item) + except httpx.HTTPError as exc: + if not is_transient(exc): + raise + elapsed = time.monotonic() - started + error = f"{type(exc).__name__}: {exc}" + evaluation = ItemEvaluation(passed=False, rank_key=(-2, 0.0), detail={"request_error": error}, error=error) + return _CatalogRun(ChatResult(), evaluation, elapsed, PhaseTimings(agent_s=elapsed), request_error=exc) + asked = time.monotonic() + evaluation = evaluator.evaluate(item, chat_result) + judged = time.monotonic() + return _CatalogRun( + chat_result, evaluation, asked - started, PhaseTimings(agent_s=asked - started, judge_s=judged - asked) + ) + + +def evaluate_catalog_metadata( + *, + host: str, + token: str, + workspace_id: str, + dataset_item: dict, + dataset_name: str, + run_timestamp: str | None, + model_version_override: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + k: int = 1, + gate: EvalGate = DEFAULT_GATE, + run_metadata_extra: dict | None = None, + langfuse: object | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + client: CatalogMetadataClient | None = None, + evaluator: CatalogMetadataEvaluator | None = None, +) -> AgenticEvalOutcome: + """Generate the item's title or description K times, grade each, log to Langfuse, and raise on a failed gate. + + Args: + host: GoodData host URL. + token: API token. + workspace_id: Workspace the catalog object lives in. + dataset_item: The item as stored: `id`, `question`, `expected_output`, `catalog_input`. + dataset_name: Langfuse dataset name; prefixes the experiment run name. + run_timestamp: Shared run timestamp for the run name. + model_version_override: Model-version tag for the run; the workspace's active LLM + provider otherwise. + reasoning_effort: Only names the run: the generate endpoints have no reasoning option. + k: Runs per item. + gate: `any` (pass@K) or `power` (pass^K). + run_metadata_extra: Extra experiment-run metadata, e.g. `testing_framework`. + langfuse: Langfuse client; built from the LANGFUSE_* environment when None. + submit_trace_link: How the Langfuse block is run; inline by default. + client: Catalog client to use instead of one built from host/token/workspace. + evaluator: Evaluator to use instead of a default one. + + Raises: + CatalogMetadataAssertionError: the gate did not pass. + ValueError: the item has no `catalog_input` or an `expected_output` the evaluator + cannot read, raised before any request. + httpx.HTTPError: the endpoint refused with a non-transient 4xx, or no run completed; + the message carries status and body. + JudgeResponseError: the judge returned no readable verdict for any run. + """ + item = _dataset_item(dataset_item, dataset_name) + evaluator = evaluator or CatalogMetadataEvaluator() + evaluator.validate(item) + own_client = client is None + catalog_client = client or CatalogMetadataClient(host=host, token=token, workspace_id=workspace_id) + langfuse, window_start = open_trace_window(langfuse) + try: + runs = [_run_once(catalog_client, evaluator, item) for _ in range(k)] + finally: + if own_client: + catalog_client.close() + + completed = [r for r in runs if r.request_error is None] + if not completed: + raise runs[-1].request_error or RuntimeError("no catalog_metadata run completed") + graded = [r for r in runs if r.judge_error is None] + pass_at_k = any(r.evaluation.passed for r in graded) + pass_power_k = all(r.evaluation.passed for r in runs) + + if langfuse is not None and item.id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(runs), gate=gate) + for run_idx, run in enumerate(runs): + if run.judge_error is not None: + continue + with ctx.observe(None, run_idx, output=run.chat_result.raw_response) as tid: + for name, value in run.checks.items(): + ctx.score(tid, name=name, value=float(value), data_type="BOOLEAN") + for name in _COUNT_SCORES: + if name in run.evaluation.detail: + ctx.score(tid, name=name, value=run.evaluation.detail[name], data_type="NUMERIC") + ctx.score( + tid, name="catalog_metadata_pass", value=float(run.evaluation.passed), data_type="BOOLEAN" + ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k) + ctx.quality(tid, strict_checks=run.checks, latency_sec=run.latency_s, cost_usd=None) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=item.id, + conversation_ids=[], + window_start=window_start, + window_end=window_end, + suffix_runs=len(runs) > 1, + write_scores=_write_scores, + item_input=item.question, + ) + + item_timings = sum_timings([r.timings for r in runs]) + unscored = [r.judge_error for r in runs if r.judge_error is not None] + request_errors = [r.judge_error for r in runs if r.request_error is not None and r.judge_error is not None] + judge_errors = [r.judge_error for r in completed if r.judge_error is not None] + best = max(completed, key=lambda r: r.evaluation.rank_key) + runs_passed = sum(1 for r in runs if r.evaluation.passed) + detail: dict[str, Any] = { + **best.evaluation.detail, + **({"unscored_runs": len(unscored)} if unscored else {}), + **({"judge_errors": judge_errors} if judge_errors else {}), + **({"request_errors": request_errors} if request_errors else {}), + } + failed_runs = build_failed_runs(runs, passed=lambda r: r.evaluation.passed, detail=lambda r: r.evaluation.detail) + + def _attach(exc: CatalogMetadataAssertionError | JudgeResponseError) -> None: + exc.reasoning_steps = [] + exc.conversation_id = "" + exc.response_id = None + exc.timings = item_timings + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = len(runs) + exc.failed_runs = failed_runs + + if not graded: + if request_errors: + causes = [ + *([f"judge returned no readable verdict for {len(judge_errors)} run(s)"] if judge_errors else []), + f"request failed for {len(request_errors)} run(s)", + ] + message = f"no run of {len(runs)} could be graded: {', '.join(causes)}: " + else: + message = f"judge returned no readable verdict for any of the {len(runs)} run(s): " + exc_judge = JudgeResponseError(message + " | ".join(unscored)) + _attach(exc_judge) + raise exc_judge + + if not gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k): + failed = [name for name, ok in best.checks.items() if not ok] + parts = [ + "Catalog metadata assertion failed.", + gate_failure_note(gate, runs_passed, len(runs), len(unscored)), + *([f"Failed checks: {', '.join(failed)}."] if failed else []), + *([f"Request errors: {' | '.join(request_errors)}"] if request_errors else []), + f"Generated: {best.chat_result.text_response!r}", + ] + exc = CatalogMetadataAssertionError(" ".join(parts)) + _attach(exc) + raise exc + return AgenticEvalOutcome( + detail=detail, + timings=item_timings, + runs_passed=runs_passed, + runs_effective=len(runs), + failed_runs=failed_runs, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py b/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py new file mode 100644 index 000000000..f6ff13dfa --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py @@ -0,0 +1,97 @@ +# (C) 2026 GoodData Corporation +"""HTTP client for the Analytics Catalog title/description generation endpoints. + +Endpoints (afm-exec-api, served by gen-ai): + POST /api/v1/actions/workspaces/{workspace_id}/ai/analyticsCatalog/generateTitle + POST /api/v1/actions/workspaces/{workspace_id}/ai/analyticsCatalog/generateDescription + +Both take ``{"objectType", "objectId"}`` and answer ``{"title"|"description", "note"}``. +The generated text becomes ``ChatResult.text_response``; the whole response body is kept +on ``raw_response`` with two extra keys the evaluator reads: ``output`` (which field was +asked for) and ``unusable_retries``. A placeholder ``"string"`` is dropped from both the +text and the note. + +A 200 answer can carry no usable text: a null value with a null note, or the literal +schema placeholder ``"string"``. The endpoint is asked again once in that case, and the +retry is recorded, so a run reports the answer a user gets after one regenerate click. +A null value that comes with a note is the endpoint declining on purpose and is kept. +""" + +from typing import Any + +import httpx + +from gooddata_eval.core._retry import is_retryable_request_error, retry_transient +from gooddata_eval.core.models import CatalogInput, ChatResult, DatasetItem + +_PATH = "/api/v1/actions/workspaces/{workspace_id}/ai/analyticsCatalog/{action}" +_ACTIONS = {"title": "generateTitle", "description": "generateDescription"} +_PLACEHOLDER = "string" + + +def is_unusable(data: dict[str, Any], output: str) -> bool: + """Whether a 200 answer carries neither a generated text nor a reason for its absence.""" + value = data.get(output) + note = data.get("note") + if value == _PLACEHOLDER: + return True + return not value and not note + + +class CatalogMetadataClient: + """Single-shot client for the Analytics Catalog generate endpoints.""" + + def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 120.0, unusable_retries: int = 1): + self._base = host.rstrip("/") + self._workspace_id = workspace_id + self._auth = {"Authorization": f"Bearer {token}"} + self._client = httpx.Client(timeout=timeout) + self._unusable_retries = unusable_retries + + def ask(self, item: DatasetItem) -> ChatResult: + """Generate the title or description an item asks for and adapt it to a ChatResult. + + Raises: + ValueError: The item carries no `catalog_input`. + httpx.HTTPStatusError: As `request`. + """ + ci = item.catalog_input + if ci is None: + raise ValueError(f"catalog_metadata item '{item.id}' is missing required 'catalog_input'.") + return self.request(ci) + + def request(self, ci: CatalogInput) -> ChatResult: + """Generate one text and adapt it to a ChatResult. + + Raises: + httpx.HTTPStatusError: The endpoint answered non-2xx after the transient retries. + """ + url = f"{self._base}{_PATH.format(workspace_id=self._workspace_id, action=_ACTIONS[ci.output])}" + body = {"objectType": ci.object_type, "objectId": ci.object_id} + + def _post() -> httpx.Response: + resp = self._client.post(url, json=body, headers={**self._auth, "Content-Type": "application/json"}) + if not resp.is_success: + raise httpx.HTTPStatusError( + f"Catalog generate request failed with {resp.status_code} for {url}: {resp.text}", + request=resp.request, + response=resp, + ) + return resp + + retries = 0 + data: dict[str, Any] = retry_transient(_post, is_retryable=is_retryable_request_error).json() + while is_unusable(data, ci.output) and retries < self._unusable_retries: + retries += 1 + data = retry_transient(_post, is_retryable=is_retryable_request_error).json() + value = data.get(ci.output) + text = value if isinstance(value, str) and value != _PLACEHOLDER else "" + # The placeholder is no reason either: a declined answer must carry a real note. + note = data.get("note") if data.get("note") != _PLACEHOLDER else None + return ChatResult( + text_response=text, + raw_response={**data, "note": note, "output": ci.output, "unusable_retries": retries}, + ) + + def close(self) -> None: + self._client.close() diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index 0c9047460..878bac8e7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -18,7 +18,7 @@ from pydantic import ValidationError from gooddata_eval.core.langfuse._env import make_http_client -from gooddata_eval.core.models import DatasetItem, SummaryInput, TakeawayInput +from gooddata_eval.core.models import CatalogInput, DatasetItem, SummaryInput, TakeawayInput _PAGE_SIZE = 100 @@ -60,6 +60,27 @@ def _summary_input_from_raw(raw: dict, expected_output: Any) -> SummaryInput | N return SummaryInput.model_validate(candidate) if candidate is not None else None +def _catalog_input_from_raw(raw: dict, expected_output: Any) -> CatalogInput | None: + """Locate a catalog_metadata item's `catalog_input`. + + Raises: + ValueError: The value found is not a valid `catalog_input` object. + """ + for source in (raw.get("input"), raw.get("metadata"), expected_output): + if not isinstance(source, dict) or source.get("catalog_input") is None: + continue + found = source["catalog_input"] + if not isinstance(found, dict): + raise ValueError( + f"Langfuse item {raw.get('id')!r}: catalog_input must be a JSON object, got {type(found).__name__}" + ) + try: + return CatalogInput.model_validate(found) + except ValidationError as e: + raise ValueError(f"Langfuse item {raw.get('id')!r}: invalid catalog_input: {e}") from e + return None + + def _takeaway_input_from_raw(raw: dict, expected_output: Any) -> TakeawayInput | None: """Locate a document_takeaway item's `takeaway_input`. @@ -138,9 +159,14 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt # REST API returns camelCase: expectedOutput, not expected_output expected_output = raw.get("expectedOutput") or raw.get("expected_output") takeaway_input = _takeaway_input_from_raw(raw, expected_output) - # An item carrying `takeaway_input` can only be a takeaway item, so it stands in for - # the CLI default; an explicit test_kind or a structural match still wins. - default_kind = "document_takeaway" if takeaway_input is not None else test_kind + catalog_input = _catalog_input_from_raw(raw, expected_output) + # An item carrying `takeaway_input` or `catalog_input` can only be of that kind, so it stands + # in for the CLI default; an explicit test_kind or a structural match still wins. + default_kind = test_kind + if catalog_input is not None: + default_kind = "catalog_metadata" + if takeaway_input is not None: + default_kind = "document_takeaway" resolved_kind = _infer_test_kind(expected_output, default_kind, raw.get("metadata")) return DatasetItem( id=str(raw["id"]), @@ -150,6 +176,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt expected_output=expected_output, summary_input=_summary_input_from_raw(raw, expected_output), takeaway_input=takeaway_input, + catalog_input=catalog_input, user_context=_user_context_from_raw(raw), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py index 16a9e91b7..7e2091619 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py @@ -21,7 +21,7 @@ } # LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question, -# document_takeaway) +# document_takeaway, catalog_metadata) # require the [llm-judge] extra. Their modules are imported lazily on first use so the CLI # starts without openai. # @@ -36,6 +36,7 @@ "dashboard_summary": "gooddata_eval.core.evaluators.summary", "knowledge_question": "gooddata_eval.core.evaluators.general_question", "document_takeaway": "gooddata_eval.core.evaluators.takeaway", + "catalog_metadata": "gooddata_eval.core.evaluators.catalog_metadata", } _LAZY_EVALUATOR_CLASSES: dict[str, str] = { "general_question": "GeneralQuestionEvaluator", @@ -43,6 +44,7 @@ "dashboard_summary": "DashboardSummaryEvaluator", "knowledge_question": "GeneralQuestionEvaluator", "document_takeaway": "DocumentTakeawayEvaluator", + "catalog_metadata": "CatalogMetadataEvaluator", } @@ -69,7 +71,7 @@ def supported_test_kinds() -> set[str]: """Return all supported test_kind values. LLM-judge kinds (general_question, guardrail, dashboard_summary, - knowledge_question, document_takeaway) are excluded when the [llm-judge] extra (openai) is + knowledge_question, document_takeaway, catalog_metadata) are excluded when the [llm-judge] extra (openai) is not installed — those items are skipped rather than erroring out mid-run. """ kinds = set(_EAGER_EVALUATORS) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py new file mode 100644 index 000000000..f74c7db09 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py @@ -0,0 +1,162 @@ +# (C) 2026 GoodData Corporation +"""Evaluator for catalog_metadata: format checks plus an LLM-judge rubric on a generated title or description. + +The Analytics Catalog generates a title (a short label) or a description (one to three +plain sentences) for one object from its metadata. The default limits (title 10 words, +description 4 sentences) sit one step above what the product prompt asks for (2-8 words, +1-3 sentences): they catch a broken shape, not a slightly long answer. `expected_output` is: + + { + "expect_note": false, + "format": {...overrides of the output's default format...}, + "must_include": ["...criteria the text must satisfy..."], + "must_not_include": ["...characteristics the text must not show..."], + "rubric": ["...soft quality dimensions..."] + } + +Every key is optional. The format checks need no judge and gate the verdict; a ``format`` +value of ``false`` or ``null`` switches its check off. ``expect_note`` decides the item +alone: the endpoint must return no text and a non-empty note (it declined, e.g. the +object does not exist). Judge criteria grade like ``dashboard_summary``; the item's +question is what the judge reads as INPUT, so it states the object's real metadata in +plain words. +""" + +import re +from typing import Any + +from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.takeaway import _sentences, _words +from gooddata_eval.core.models import ChatResult, DatasetItem + +_DEFAULT_FORMAT: dict[str, dict[str, Any]] = { + "title": {"max_words": 10, "max_sentences": 1, "no_end_period": True, "no_markup": True, "no_ids": True}, + "description": {"max_words": 80, "max_sentences": 4, "no_end_period": False, "no_markup": True, "no_ids": True}, +} + +_POSITIVE_STEPS = [ + "Read the INPUT (the analytics object the text was generated for, with its real metadata) and the " + "EXPECTED OUTPUT (one criterion the text must satisfy).", + "Read the ACTUAL OUTPUT (the generated title or description).", + "Score 1 if the actual output clearly satisfies the criterion.", + "Score 0 if the criterion is missing, contradicted, or only partially addressed.", +] + +# A presence question, inverted in code -- see the same split in summary.py. The INPUT +# carries the object's real metadata, which pulls the judge towards grading correctness; +# the third step keeps it on presence. +_VIOLATION_STEPS = [ + "Read the CHARACTERISTIC described in EXPECTED OUTPUT, including any exceptions it states.", + "Read the ACTUAL OUTPUT (the generated title or description).", + "Decide only whether the actual output exhibits the characteristic, not whether it is correct or wanted.", + "Score 1 if the actual output clearly exhibits the described characteristic.", + "Score 0 if it does not exhibit it.", +] + +_MARKUP = re.compile(r"\*\*|`|^\s*#{1,6}\s|^\s*(?:[-*•]|\d+[.)])\s", re.MULTILINE) +# A UUID, a MAQL reference such as {metric/net_sales}, or a snake_case identifier. +_IDS = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|\{\w+/[^}]*\}|\b[a-z0-9]+(?:_[a-z0-9]+)+\b", + re.IGNORECASE, +) + + +def _limit_on(value: Any) -> bool: + return value is not None and value is not False + + +class CatalogMetadataEvaluator: + test_kind = "catalog_metadata" + + def __init__(self): + self._positive_judge = LLMJudge(evaluation_steps=_POSITIVE_STEPS) + self._violation_judge = LLMJudge(evaluation_steps=_VIOLATION_STEPS) + + @staticmethod + def validate(item: DatasetItem) -> None: + """Reject an item this evaluator cannot read, before any request. + + Raises: + ValueError: No `catalog_input`, `format` is not an object or has an unknown key, + or a criteria key is not a JSON array. + """ + if item.catalog_input is None: + raise ValueError(f"catalog_metadata item '{item.id}' is missing required 'catalog_input'") + expected = item.expected_output if isinstance(item.expected_output, dict) else {} + overrides = expected.get("format") or {} + if not isinstance(overrides, dict): + raise ValueError(f"catalog_metadata item '{item.id}': format must be a JSON object") + known = _DEFAULT_FORMAT[item.catalog_input.output] + unknown = sorted(set(overrides) - set(known)) + if unknown: + raise ValueError( + f"catalog_metadata item '{item.id}': unknown format key(s) {unknown}; known: {sorted(known)}" + ) + for key in ("must_include", "must_not_include", "rubric"): + if not isinstance(expected.get(key, []), list): + raise ValueError(f"catalog_metadata item '{item.id}': {key} must be a JSON array") + + def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation: + self.validate(item) + assert item.catalog_input is not None # validate() guarantees it + output = item.catalog_input.output + expected: dict[str, Any] = item.expected_output if isinstance(item.expected_output, dict) else {} + raw = chat_result.raw_response or {} + text = (chat_result.text_response or "").strip() + note = raw.get("note") or "" + detail: dict[str, Any] = { + "actual_output": text, + "note": note, + "unusable_retries": raw.get("unusable_retries", 0), + } + + if expected.get("expect_note") is True: + ok = not text and bool(note.strip()) + detail["declined_with_note"] = ok + return ItemEvaluation(passed=ok, rank_key=(int(ok), float(ok)), detail=detail) + + detail["generated"] = bool(text) + detail["no_note"] = not note.strip() + if text: + detail.update(self._format_checks(output, expected, text)) + passed = all(v for v in detail.values() if isinstance(v, bool)) + if not text: + return ItemEvaluation(passed=False, rank_key=(0, 0.0), detail=detail) + + must_include: list[Criterion] = [ + (f"include_{i}", str(c)) for i, c in enumerate(expected.get("must_include", [])) + ] + must_not_include: list[Criterion] = [ + (f"exclude_{i}", str(c)) for i, c in enumerate(expected.get("must_not_include", [])) + ] + rubric: list[Criterion] = [(f"rubric_{i}", str(c)) for i, c in enumerate(expected.get("rubric", []))] + return grade_rubric( + self._positive_judge, + self._violation_judge, + question=item.question, + actual=text, + must_include=must_include, + must_not_include=must_not_include, + rubric=rubric, + detail=detail, + passed=passed, + ) + + @staticmethod + def _format_checks(output: str, expected: dict[str, Any], text: str) -> dict[str, Any]: + """Deterministic checks keyed by name, plus `word_count`.""" + fmt = {**_DEFAULT_FORMAT[output], **(expected.get("format") or {})} + checks: dict[str, Any] = {"word_count": _words(text)} + if _limit_on(fmt["max_words"]): + checks["max_words"] = checks["word_count"] <= fmt["max_words"] + if _limit_on(fmt["max_sentences"]): + checks["max_sentences"] = _sentences(text) <= fmt["max_sentences"] + if fmt["no_end_period"]: + checks["no_end_period"] = not text.endswith(".") + if fmt["no_markup"]: + checks["no_markup"] = not _MARKUP.search(text) + if fmt["no_ids"]: + checks["no_ids"] = not _IDS.search(text) + return checks diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 391267b78..eceda9f54 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -7,7 +7,7 @@ import json import re from enum import Enum -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, field_validator, model_serializer @@ -564,6 +564,21 @@ class TakeawayInput(BaseModel): agent_id: str | None = None +class CatalogInput(BaseModel): + """Structured input for the `catalog_metadata` test kind. + + Names one Analytics Catalog object and which text to generate for it: `title` calls + `generateTitle`, `description` calls `generateDescription`. Unknown keys are rejected + so a typo in a dataset item fails loudly. + """ + + model_config = ConfigDict(extra="forbid") + + object_type: Literal["Visualization", "Dashboard", "Metric", "Fact", "Attribute"] + object_id: str = Field(min_length=1) + output: Literal["title", "description"] + + class DatasetItem(BaseModel): """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape.""" @@ -578,6 +593,8 @@ class DatasetItem(BaseModel): summary_input: SummaryInput | None = None # Only used by the `document_takeaway` test kind; ignored by all others. takeaway_input: TakeawayInput | None = None + # Only used by the `catalog_metadata` test kind; ignored by all others. + catalog_input: CatalogInput | None = None # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py index d946aa5ab..1bcb325e5 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py @@ -17,9 +17,11 @@ import httpx -from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, retry_transient +from gooddata_eval.core._retry import is_retryable_request_error, is_transient, retry_transient from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput, TakeawayVisualization +__all__ = ["TakeawayClient", "is_transient"] + _PATH = "/api/v1/ai/workspaces/{workspace_id}/takeaway" @@ -29,29 +31,6 @@ def _visualization_body(visualization: str | TakeawayVisualization) -> str | dic return {"visualizationId": visualization.visualization_id, "filters": visualization.filters} -# Failures that say nothing about the request. A read timeout is left out: the page was -# being computed, and retrying spends the whole timeout again on every attempt. -_RETRYABLE_TRANSPORT_ERRORS = (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError, httpx.PoolTimeout) - - -def _is_retryable(exc: Exception) -> bool: - if isinstance(exc, httpx.HTTPStatusError): - return exc.response.status_code in RETRYABLE_STATUS_CODES - return isinstance(exc, _RETRYABLE_TRANSPORT_ERRORS) - - -def is_transient(exc: httpx.HTTPError) -> bool: - """Whether a failure left after the client's retries says nothing about the request. - - 429, 5xx and transport failures (a read timeout included) are transient; any other 4xx - (flag off, no agent, invalid body) and an unsupported URL scheme are setup problems. - """ - if isinstance(exc, httpx.HTTPStatusError): - status = exc.response.status_code - return status == 429 or status >= 500 - return isinstance(exc, httpx.TransportError) and not isinstance(exc, httpx.UnsupportedProtocol) - - def _request_body(ti: TakeawayInput) -> dict[str, Any]: body: dict[str, Any] = {} if ti.agent_id is not None: @@ -116,7 +95,7 @@ def _post() -> httpx.Response: ) return resp - data = retry_transient(_post, is_retryable=_is_retryable).json() + data = retry_transient(_post, is_retryable=is_retryable_request_error).json() return ChatResult(text_response=data.get("takeaway") or "", raw_response=data) def close(self) -> None: diff --git a/packages/gooddata-eval/tests/fixtures/catalog_metadata_calibration.json b/packages/gooddata-eval/tests/fixtures/catalog_metadata_calibration.json new file mode 100644 index 000000000..ec4d1e7eb --- /dev/null +++ b/packages/gooddata-eval/tests/fixtures/catalog_metadata_calibration.json @@ -0,0 +1,290 @@ +[ + { + "label": "m-single-orders-desc include_0 pass", + "question": "Description for the metric '# of Single Orders'. Its MAQL sums Total Customers where # of Orders = 1, so it counts customers who placed exactly one order. No existing description.", + "output": "description", + "kind": "include", + "criterion": "Says that it counts customers (people), namely customers who placed exactly one order.", + "text": "Number of customers who placed exactly one order. This metric is useful for measuring single-order customer activity and identifying customers who have not made repeat purchases.", + "want": "pass" + }, + { + "label": "m-single-orders-desc include_0 fail", + "question": "Description for the metric '# of Single Orders'. Its MAQL sums Total Customers where # of Orders = 1, so it counts customers who placed exactly one order. No existing description.", + "output": "description", + "kind": "include", + "criterion": "Says that it counts customers (people), namely customers who placed exactly one order.", + "text": "Counts the number of single orders placed, giving a quick view of order volume.", + "want": "fail" + }, + { + "label": "m-single-orders-desc exclude_0 pass", + "question": "Description for the metric '# of Single Orders'. Its MAQL sums Total Customers where # of Orders = 1, so it counts customers who placed exactly one order. No existing description.", + "output": "description", + "kind": "exclude", + "criterion": "Says the metric counts orders (for example single orders or one-off orders) rather than customers.", + "text": "Number of customers who placed exactly one order. This metric is useful for measuring single-order customer activity and identifying customers who have not made repeat purchases.", + "want": "pass" + }, + { + "label": "m-single-orders-desc exclude_0 fail", + "question": "Description for the metric '# of Single Orders'. Its MAQL sums Total Customers where # of Orders = 1, so it counts customers who placed exactly one order. No existing description.", + "output": "description", + "kind": "exclude", + "criterion": "Says the metric counts orders (for example single orders or one-off orders) rather than customers.", + "text": "Counts orders that contain a single item, showing how many one-off orders were placed.", + "want": "fail" + }, + { + "label": "m-orders-all-time-desc include_0 pass", + "question": "Description for the metric 'Orders all time'. Its MAQL is '# of Orders WITHOUT PF', i.e. the order count ignoring the filters applied to the report. No existing description.", + "output": "description", + "kind": "include", + "criterion": "Describes it as a count of orders.", + "text": "Total number of orders across all time and statuses, unaffected by filters applied to the surrounding report or dashboard.", + "want": "pass" + }, + { + "label": "m-orders-all-time-desc include_0 fail", + "question": "Description for the metric 'Orders all time'. Its MAQL is '# of Orders WITHOUT PF', i.e. the order count ignoring the filters applied to the report. No existing description.", + "output": "description", + "kind": "include", + "criterion": "Describes it as a count of orders.", + "text": "Total revenue generated across all time.", + "want": "fail" + }, + { + "label": "m-orders-all-time-desc exclude_0 pass", + "question": "Description for the metric 'Orders all time'. Its MAQL is '# of Orders WITHOUT PF', i.e. the order count ignoring the filters applied to the report. No existing description.", + "output": "description", + "kind": "exclude", + "criterion": "States that the count covers only one period, such as the current year, a month or a recent period.", + "text": "Total number of orders across all time and statuses, unaffected by filters applied to the surrounding report or dashboard.", + "want": "pass" + }, + { + "label": "m-orders-all-time-desc exclude_0 fail", + "question": "Description for the metric 'Orders all time'. Its MAQL is '# of Orders WITHOUT PF', i.e. the order count ignoring the filters applied to the report. No existing description.", + "output": "description", + "kind": "exclude", + "criterion": "States that the count covers only one period, such as the current year, a month or a recent period.", + "text": "Counts orders placed in the current year.", + "want": "fail" + }, + { + "label": "m-return-customers-desc include_0 pass", + "question": "Description for the metric 'Return Customers'. Its MAQL is Active Customers minus New Customers, i.e. active customers who are not new (returning customers). No existing description.", + "output": "description", + "kind": "include", + "criterion": "Describes returning (repeat) customers: active customers who are not new.", + "text": "Number of returning customers in the selected period: active customers who transacted minus those placing their first order during that period.", + "want": "pass" + }, + { + "label": "m-return-customers-desc include_0 fail", + "question": "Description for the metric 'Return Customers'. Its MAQL is Active Customers minus New Customers, i.e. active customers who are not new (returning customers). No existing description.", + "output": "description", + "kind": "include", + "criterion": "Describes returning (repeat) customers: active customers who are not new.", + "text": "Counts customers who returned at least one product.", + "want": "fail" + }, + { + "label": "m-return-customers-desc exclude_0 pass", + "question": "Description for the metric 'Return Customers'. Its MAQL is Active Customers minus New Customers, i.e. active customers who are not new (returning customers). No existing description.", + "output": "description", + "kind": "exclude", + "criterion": "Describes it as customers who returned products or as product returns.", + "text": "Number of returning customers in the selected period: active customers who transacted minus those placing their first order during that period.", + "want": "pass" + }, + { + "label": "m-return-customers-desc exclude_0 fail", + "question": "Description for the metric 'Return Customers'. Its MAQL is Active Customers minus New Customers, i.e. active customers who are not new (returning customers). No existing description.", + "output": "description", + "kind": "exclude", + "criterion": "Describes it as customers who returned products or as product returns.", + "text": "Counts customers who sent back purchased products for a refund.", + "want": "fail" + }, + { + "label": "v-order-details-processed-desc include_0 pass", + "question": "Description for the visualization 'Order Details - Processed' (table of units and unit price by order id, order line, order status and product name). Its filter EXCLUDES order status 'Processed' and 'Canceled', so despite the title it shows no processed orders.", + "output": "description", + "kind": "include", + "criterion": "Says that processed and canceled orders are excluded.", + "text": "A table of order-line details showing total unit quantity and unit price by order, order line, status, and product. It excludes orders with Processed or Canceled status to focus on other order statuses.", + "want": "pass" + }, + { + "label": "v-order-details-processed-desc include_0 fail", + "question": "Description for the visualization 'Order Details - Processed' (table of units and unit price by order id, order line, order status and product name). Its filter EXCLUDES order status 'Processed' and 'Canceled', so despite the title it shows no processed orders.", + "output": "description", + "kind": "include", + "criterion": "Says that processed and canceled orders are excluded.", + "text": "Table of order lines with units and unit price by order, status and product.", + "want": "fail" + }, + { + "label": "v-order-details-processed-desc exclude_0 pass", + "question": "Description for the visualization 'Order Details - Processed' (table of units and unit price by order id, order line, order status and product name). Its filter EXCLUDES order status 'Processed' and 'Canceled', so despite the title it shows no processed orders.", + "output": "description", + "kind": "exclude", + "criterion": "Says the table shows only processed orders or is limited to processed orders.", + "text": "A table of order-line details showing total unit quantity and unit price by order, order line, status, and product. It excludes orders with Processed or Canceled status to focus on other order statuses.", + "want": "pass" + }, + { + "label": "v-order-details-processed-desc exclude_0 fail", + "question": "Description for the visualization 'Order Details - Processed' (table of units and unit price by order id, order line, order status and product name). Its filter EXCLUDES order status 'Processed' and 'Canceled', so despite the title it shows no processed orders.", + "output": "description", + "kind": "exclude", + "criterion": "Says the table shows only processed orders or is limited to processed orders.", + "text": "Table of processed orders only, showing units and unit price per order line.", + "want": "fail" + }, + { + "label": "f-order-unit-price-desc include_0 pass", + "question": "Description for the fact 'Order unit price' (dataset 'Order lines'; existing description 'Order unit price'). The metadata names no currency.", + "output": "description", + "kind": "include", + "criterion": "Describes the price of one unit on an order line.", + "text": "The price charged for each individual item on an order line in the Order lines dataset. Use it to analyze product pricing, typically with averages or other price comparisons across products, customers, or time periods.", + "want": "pass" + }, + { + "label": "f-order-unit-price-desc include_0 fail", + "question": "Description for the fact 'Order unit price' (dataset 'Order lines'; existing description 'Order unit price'). The metadata names no currency.", + "output": "description", + "kind": "include", + "criterion": "Describes the price of one unit on an order line.", + "text": "The total revenue of an order after discounts.", + "want": "fail" + }, + { + "label": "f-order-unit-price-desc exclude_0 pass", + "question": "Description for the fact 'Order unit price' (dataset 'Order lines'; existing description 'Order unit price'). The metadata names no currency.", + "output": "description", + "kind": "exclude", + "criterion": "Names a specific currency such as USD, dollars, EUR or euros.", + "text": "The price charged for each individual item on an order line in the Order lines dataset. Use it to analyze product pricing, typically with averages or other price comparisons across products, customers, or time periods.", + "want": "pass" + }, + { + "label": "f-order-unit-price-desc exclude_0 fail", + "question": "Description for the fact 'Order unit price' (dataset 'Order lines'; existing description 'Order unit price'). The metadata names no currency.", + "output": "description", + "kind": "exclude", + "criterion": "Names a specific currency such as USD, dollars, EUR or euros.", + "text": "The price of one unit on an order line, in US dollars.", + "want": "fail" + }, + { + "label": "v-seasonality-desc include_0 pass", + "question": "Description for the visualization 'Net Sales by Product Category Seasonality' (heatmap of Net Sales by Product category and month of year; filtered to last year).", + "output": "description", + "kind": "include", + "criterion": "Says it covers last year (the previous year).", + "text": "A heatmap showing net sales by product category and month of year for last year. Net sales reflect transaction amounts after discounts and returns, helping identify seasonal sales patterns across categories.", + "want": "pass" + }, + { + "label": "v-seasonality-desc include_0 fail", + "question": "Description for the visualization 'Net Sales by Product Category Seasonality' (heatmap of Net Sales by Product category and month of year; filtered to last year).", + "output": "description", + "kind": "include", + "criterion": "Says it covers last year (the previous year).", + "text": "Heatmap of net sales by product category and month of year.", + "want": "fail" + }, + { + "label": "v-seasonality-desc exclude_0 pass", + "question": "Description for the visualization 'Net Sales by Product Category Seasonality' (heatmap of Net Sales by Product category and month of year; filtered to last year).", + "output": "description", + "kind": "exclude", + "criterion": "Says it covers this year, the current year or all time.", + "text": "A heatmap showing net sales by product category and month of year for last year. Net sales reflect transaction amounts after discounts and returns, helping identify seasonal sales patterns across categories.", + "want": "pass" + }, + { + "label": "v-seasonality-desc exclude_0 fail", + "question": "Description for the visualization 'Net Sales by Product Category Seasonality' (heatmap of Net Sales by Product category and month of year; filtered to last year).", + "output": "description", + "kind": "exclude", + "criterion": "Says it covers this year, the current year or all time.", + "text": "Heatmap of net sales by product category and month for the current year.", + "want": "fail" + }, + { + "label": "a-quarter-of-year-desc include_0 pass", + "question": "Description for the attribute 'Order date - Quarter of Year' (dataset 'Order date'; existing description 'Generic Quarter (Q1-Q4)'): the quarter Q1-Q4 regardless of year.", + "output": "description", + "kind": "include", + "criterion": "Describes the quarter of the order date (Q1-Q4).", + "text": "Represents the quarter of the year in which an order date falls (Q1–Q4), enabling order analysis and filtering by seasonal quarter.", + "want": "pass" + }, + { + "label": "a-quarter-of-year-desc include_0 fail", + "question": "Description for the attribute 'Order date - Quarter of Year' (dataset 'Order date'; existing description 'Generic Quarter (Q1-Q4)'): the quarter Q1-Q4 regardless of year.", + "output": "description", + "kind": "include", + "criterion": "Describes the quarter of the order date (Q1-Q4).", + "text": "The month of the year in which the order was placed.", + "want": "fail" + }, + { + "label": "a-quarter-of-year-desc exclude_0 pass", + "question": "Description for the attribute 'Order date - Quarter of Year' (dataset 'Order date'; existing description 'Generic Quarter (Q1-Q4)'): the quarter Q1-Q4 regardless of year.", + "output": "description", + "kind": "exclude", + "criterion": "Says it identifies a specific quarter of a specific year (such as Q1 2024) rather than the quarter regardless of year.", + "text": "Represents the quarter of the year in which an order date falls (Q1–Q4), enabling order analysis and filtering by seasonal quarter.", + "want": "pass" + }, + { + "label": "a-quarter-of-year-desc exclude_0 fail", + "question": "Description for the attribute 'Order date - Quarter of Year' (dataset 'Order date'; existing description 'Generic Quarter (Q1-Q4)'): the quarter Q1-Q4 regardless of year.", + "output": "description", + "kind": "exclude", + "criterion": "Says it identifies a specific quarter of a specific year (such as Q1 2024) rather than the quarter regardless of year.", + "text": "The quarter and year of the order date, such as Q1 2024.", + "want": "fail" + }, + { + "label": "f-return-unit-paid-amount-title include_0 pass", + "question": "Title for the fact 'Return unit paid amount' (dataset 'Returns'; existing description 'Return unit paid amount'): the amount paid per unit for a returned item.", + "output": "title", + "kind": "include", + "criterion": "The title refers to returns (returned items).", + "text": "Return Paid Amount per Unit", + "want": "pass" + }, + { + "label": "f-return-unit-paid-amount-title include_0 fail", + "question": "Title for the fact 'Return unit paid amount' (dataset 'Returns'; existing description 'Return unit paid amount'): the amount paid per unit for a returned item.", + "output": "title", + "kind": "include", + "criterion": "The title refers to returns (returned items).", + "text": "Unit Paid Amount", + "want": "fail" + }, + { + "label": "f-return-unit-paid-amount-title exclude_0 pass", + "question": "Title for the fact 'Return unit paid amount' (dataset 'Returns'; existing description 'Return unit paid amount'): the amount paid per unit for a returned item.", + "output": "title", + "kind": "exclude", + "criterion": "The title reads as an order or sales price rather than an amount related to returns.", + "text": "Return Paid Amount per Unit", + "want": "pass" + }, + { + "label": "f-return-unit-paid-amount-title exclude_0 fail", + "question": "Title for the fact 'Return unit paid amount' (dataset 'Returns'; existing description 'Return unit paid amount'): the amount paid per unit for a returned item.", + "output": "title", + "kind": "exclude", + "criterion": "The title reads as an order or sales price rather than an amount related to returns.", + "text": "Order Unit Selling Price", + "want": "fail" + } +] \ No newline at end of file diff --git a/packages/gooddata-eval/tests/test_catalog_metadata_client.py b/packages/gooddata-eval/tests/test_catalog_metadata_client.py new file mode 100644 index 000000000..8cb4f868e --- /dev/null +++ b/packages/gooddata-eval/tests/test_catalog_metadata_client.py @@ -0,0 +1,177 @@ +# (C) 2026 GoodData Corporation +import json +from collections.abc import Callable + +import httpx +import pytest +from gooddata_eval.core import _retry as retry_mod +from gooddata_eval.core.catalog.http_client import CatalogMetadataClient, is_unusable +from gooddata_eval.core.models import CatalogInput, DatasetItem + + +def _item(catalog_input: dict | None) -> DatasetItem: + return DatasetItem( + id="c1", + dataset_name="d", + test_kind="catalog_metadata", + question="Title for the metric 'ARPU'", + expected_output={}, + catalog_input=CatalogInput.model_validate(catalog_input) if catalog_input is not None else None, + ) + + +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> CatalogMetadataClient: + client = CatalogMetadataClient(host="https://h/", token="tok", workspace_id="ws") + client._client = httpx.Client(transport=httpx.MockTransport(handler)) + return client + + +def _sequence(captured: list, responses: list[dict]) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + captured.append({"url": str(request.url), "body": json.loads(request.content)}) + return httpx.Response(200, json=responses[min(len(captured), len(responses)) - 1]) + + return handler + + +@pytest.mark.parametrize(("output", "action"), [("title", "generateTitle"), ("description", "generateDescription")]) +def test_posts_object_to_the_action_of_the_output(output: str, action: str) -> None: + captured: list = [] + client = _client(_sequence(captured, [{output: "Average Revenue per Customer", "note": None}])) + + result = client.ask(_item({"object_type": "Metric", "object_id": "arpu", "output": output})) + + assert captured == [ + { + "url": f"https://h/api/v1/actions/workspaces/ws/ai/analyticsCatalog/{action}", + "body": {"objectType": "Metric", "objectId": "arpu"}, + } + ] + assert result.text_response == "Average Revenue per Customer" + assert result.raw_response == { + output: "Average Revenue per Customer", + "note": None, + "output": output, + "unusable_retries": 0, + } + + +@pytest.mark.parametrize("unusable", [{"title": None, "note": None}, {"title": "string", "note": "string"}]) +def test_asks_again_once_after_an_unusable_answer(unusable: dict) -> None: + captured: list = [] + client = _client(_sequence(captured, [unusable, {"title": "Average Revenue", "note": None}])) + + result = client.ask(_item({"object_type": "Metric", "object_id": "arpu", "output": "title"})) + + assert len(captured) == 2 + assert result.text_response == "Average Revenue" + assert result.raw_response is not None + assert result.raw_response["unusable_retries"] == 1 + + +def test_keeps_a_second_unusable_answer_as_empty_text() -> None: + captured: list = [] + client = _client(_sequence(captured, [{"title": "string", "note": "string"}])) + + result = client.ask(_item({"object_type": "Metric", "object_id": "arpu", "output": "title"})) + + assert len(captured) == 2 + assert result.text_response == "" + + +def test_a_declined_answer_with_a_note_is_not_retried() -> None: + captured: list = [] + client = _client(_sequence(captured, [{"description": None, "note": "Object not found"}])) + + result = client.ask(_item({"object_type": "Visualization", "object_id": "x", "output": "description"})) + + assert len(captured) == 1 + assert result.text_response == "" + assert result.raw_response is not None + assert result.raw_response["note"] == "Object not found" + + +def test_rejects_an_item_without_catalog_input() -> None: + client = _client(_sequence([], [{}])) + with pytest.raises(ValueError, match="catalog_input"): + client.ask(_item(None)) + + +def test_non_2xx_raises_with_status() -> None: + client = _client(lambda request: httpx.Response(403, text="forbidden")) + with pytest.raises(httpx.HTTPStatusError, match="403"): + client.ask(_item({"object_type": "Metric", "object_id": "arpu", "output": "title"})) + + +def test_is_unusable() -> None: + assert is_unusable({"title": None, "note": None}, "title") + assert is_unusable({"title": "string", "note": None}, "title") + assert not is_unusable({"title": None, "note": "Not found"}, "title") + assert not is_unusable({"title": "Revenue", "note": None}, "title") + + +def test_a_placeholder_note_is_dropped() -> None: + client = _client(_sequence([], [{"description": None, "note": "string"}])) + + result = client.ask(_item({"object_type": "Visualization", "object_id": "x", "output": "description"})) + + assert result.text_response == "" + assert result.raw_response is not None + assert result.raw_response["note"] is None + + +@pytest.fixture +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]: + for name in ( + "GOODDATA_EVAL_CHAT_MAX_RETRIES", + "GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", + "GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", + "GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", + ): + monkeypatch.delenv(name, raising=False) + recorded: list[float] = [] + monkeypatch.setattr(retry_mod.time, "sleep", recorded.append) + return recorded + + +def _scripted(*responses: httpx.Response | Exception) -> tuple[Callable[[httpx.Request], httpx.Response], list[int]]: + calls: list[int] = [] + it = iter(responses) + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(1) + nxt = next(it) + if isinstance(nxt, Exception): + raise nxt + return nxt + + return handler, calls + + +_OK = httpx.Response(200, json={"title": "Average Revenue", "note": None}) +_ARPU_TITLE = {"object_type": "Metric", "object_id": "arpu", "output": "title"} + + +@pytest.mark.parametrize("first", [httpx.Response(503, text="busy"), httpx.ConnectError("refused")]) +def test_a_transient_failure_is_retried(first: httpx.Response | Exception, sleeps: list[float]) -> None: + handler, calls = _scripted(first, _OK) + + result = _client(handler).ask(_item(_ARPU_TITLE)) + + assert result.text_response == "Average Revenue" + assert len(calls) == 2 + assert len(sleeps) == 1 + + +def test_a_read_timeout_is_not_retried(sleeps: list[float]) -> None: + handler, calls = _scripted(httpx.ReadTimeout("read timed out"), _OK) + + with pytest.raises(httpx.ReadTimeout): + _client(handler).ask(_item(_ARPU_TITLE)) + + assert len(calls) == 1 + assert sleeps == [] + + +def test_default_timeout_is_two_minutes() -> None: + assert CatalogMetadataClient(host="https://h", token="tok", workspace_id="ws")._client.timeout.read == 120.0 diff --git a/packages/gooddata-eval/tests/test_catalog_metadata_evaluate.py b/packages/gooddata-eval/tests/test_catalog_metadata_evaluate.py new file mode 100644 index 000000000..062c4173a --- /dev/null +++ b/packages/gooddata-eval/tests/test_catalog_metadata_evaluate.py @@ -0,0 +1,256 @@ +# (C) 2026 GoodData Corporation +import inspect +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.core.agentic import _langfuse as agentic_langfuse +from gooddata_eval.core.agentic._trace_linker import run_trace_link_inline +from gooddata_eval.core.catalog.evaluate import CatalogMetadataAssertionError, evaluate_catalog_metadata +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError +from gooddata_eval.core.evaluators.catalog_metadata import CatalogMetadataEvaluator +from gooddata_eval.core.models import ChatResult + +from tests._fake_langfuse import FakeLangfuse + +_GOOD = "Shows net sales by product category over the last 12 months, so teams can compare category performance." +_LONG = "Sales grew. " + " ".join(["word"] * 90) + "." +_SCORES = "/api/public/scores" + + +def _posted_scores(fake_langfuse: FakeLangfuse) -> list[dict]: + bodies: list[dict] = [] + for call in fake_langfuse.calls("POST", _SCORES): + body = call["json"] + bodies.extend(body if isinstance(body, list) else [body]) + return bodies + + +def _dataset_item(**overrides: Any) -> dict: + return { + "id": "item-1", + "question": "Description for the visualization 'Net Sales by Product Category'", + "expected_output": {"must_include": ["Mentions product categories."]}, + "catalog_input": {"object_type": "Visualization", "object_id": "v1", "output": "description"}, + **overrides, + } + + +def _chat(text: str, retries: int = 0) -> ChatResult: + return ChatResult( + text_response=text, + raw_response={"description": text, "note": None, "output": "description", "unusable_retries": retries}, + ) + + +def _client(*texts: str) -> MagicMock: + client = MagicMock() + client.ask.side_effect = [_chat(t) for t in texts] + return client + + +def _evaluator(*, judge_error: bool = False) -> CatalogMetadataEvaluator: + with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): + ev = CatalogMetadataEvaluator() + for judge, verdict in ((ev._positive_judge, True), (ev._violation_judge, False)): + judge.score = MagicMock( + side_effect=JudgeResponseError("empty body twice") if judge_error else None, return_value=(verdict, "ok") + ) + return ev + + +def _evaluate(client: MagicMock | None, **kwargs: Any) -> Any: + params: dict[str, Any] = { + "host": "https://h", + "token": "tok", + "workspace_id": "ws", + "dataset_item": _dataset_item(), + "dataset_name": "analytics_catalog_generate", + "run_timestamp": "2026-11-17_00-00-00", + "client": client, + "evaluator": _evaluator(), + "langfuse": MagicMock(), + **kwargs, + } + return evaluate_catalog_metadata(**params) + + +def _http_error(status: int, body: str) -> httpx.HTTPStatusError: + request = httpx.Request("POST", "https://h/api/v1/actions/workspaces/ws/ai/analyticsCatalog/generateDescription") + response = httpx.Response(status, request=request, text=body) + return httpx.HTTPStatusError( + f"Catalog generate request failed with {status}: {body}", request=request, response=response + ) + + +def test_evaluate_builds_and_closes_its_own_client_when_none_is_given() -> None: + client = _client(_GOOD) + with patch("gooddata_eval.core.catalog.evaluate.CatalogMetadataClient", return_value=client) as client_cls: + outcome = _evaluate(None) + + client_cls.assert_called_once_with(host="https://h", token="tok", workspace_id="ws") + client.close.assert_called_once() + assert outcome.runs_passed == 1 + + +def test_evaluate_passes_and_returns_the_outcome() -> None: + outcome = _evaluate(_client(_GOOD, _GOOD), k=2) + + assert outcome.runs_passed == 2 + assert outcome.runs_effective == 2 + assert outcome.detail["word_count"] == 17 + assert outcome.failed_runs == [] + + +def test_evaluate_raises_on_fail_with_detail() -> None: + with pytest.raises(CatalogMetadataAssertionError) as exc_info: + _evaluate(_client(_LONG)) + + exc = exc_info.value + assert "max_words" in str(exc) + assert "Gate pass@1 failed: 0/1 runs passed" in str(exc) + assert exc.detail["max_words"] is False + assert exc.runs_passed == 0 + assert len(exc.failed_runs) == 1 + + +def test_evaluate_power_gate_fails_on_one_bad_run() -> None: + with pytest.raises(CatalogMetadataAssertionError, match="pass\\^2 failed: 1/2"): + _evaluate(_client(_GOOD, _LONG), k=2, gate="power") + + outcome = _evaluate(_client(_GOOD, _LONG), k=2, gate="any") + assert outcome.runs_passed == 1 + + +def test_every_run_ungraded_is_a_judge_error_not_a_failure() -> None: + with pytest.raises(JudgeResponseError, match="no readable verdict") as exc_info: + _evaluate(_client(_GOOD, _GOOD), k=2, evaluator=_evaluator(judge_error=True)) + + assert exc_info.value.detail["unscored_runs"] == 2 + + +def test_item_without_catalog_input_is_rejected_before_any_request() -> None: + client = _client(_GOOD) + item = _dataset_item() + del item["catalog_input"] + + with pytest.raises(ValueError, match="'item-1'.*catalog_input"): + _evaluate(client, dataset_item=item) + + client.ask.assert_not_called() + + +def test_an_invalid_fixture_fails_before_any_request() -> None: + client = _client(_GOOD) + + with pytest.raises(ValueError, match="max_word"): + _evaluate(client, dataset_item=_dataset_item(expected_output={"format": {"max_word": 1}})) + + client.ask.assert_not_called() + + +def test_signature_accepts_what_the_tavern_shim_passes() -> None: + params = inspect.signature(evaluate_catalog_metadata).parameters + + for name in ( + "host", + "token", + "workspace_id", + "dataset_item", + "dataset_name", + "run_timestamp", + "model_version_override", + "reasoning_effort", + "k", + "gate", + "run_metadata_extra", + ): + assert name in params, name + + +def test_scores_are_written_per_run(fake_langfuse: FakeLangfuse) -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD, retries=1), _chat(_LONG)] + with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"): + _evaluate(client, k=2, gate="any", langfuse=None, submit_trace_link=run_trace_link_inline) + + scores = _posted_scores(fake_langfuse) + passes = [s for s in scores if s["name"] == "catalog_metadata_pass"] + assert sorted(s["value"] for s in passes) == [0, 1] + names = {s["name"] for s in scores} + for name in ("gate_passed", "pass_at_k", "generated", "no_note", "max_words", "include_0", "quality_score"): + assert name in names, name + assert sorted(s["value"] for s in scores if s["name"] == "unusable_retries") == [0, 1] + + +def test_a_transient_error_on_one_run_keeps_the_graded_runs() -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD), _http_error(502, "bad gateway")] + + outcome = _evaluate(client, k=2, gate="any") + + assert outcome.runs_passed == 1 + assert "502" in outcome.detail["request_errors"][0] + + +def test_a_transient_error_on_every_run_raises_it() -> None: + client = MagicMock() + client.ask.side_effect = [_http_error(502, "bad gateway")] * 2 + + with pytest.raises(httpx.HTTPStatusError, match="502"): + _evaluate(client, k=2) + + +def test_a_deterministic_client_error_raises_on_the_first_run() -> None: + client = MagicMock() + client.ask.side_effect = [_http_error(403, "AI module not entitled"), _chat(_GOOD)] + + with pytest.raises(httpx.HTTPStatusError, match="not entitled"): + _evaluate(client, k=2) + + assert client.ask.call_count == 1 + + +def test_evaluate_leaves_an_injected_client_open() -> None: + client = _client(_GOOD) + _evaluate(client) + client.close.assert_not_called() + + +def test_a_read_timeout_is_an_errored_run_not_an_item_failure() -> None: + client = MagicMock() + client.ask.side_effect = [_chat(_GOOD), httpx.ReadTimeout("read timed out")] + + outcome = _evaluate(client, k=2, gate="any") + + assert outcome.runs_passed == 1 + assert "ReadTimeout" in outcome.detail["request_errors"][0] + + +def test_an_unsupported_protocol_raises_on_the_first_run() -> None: + client = MagicMock() + client.ask.side_effect = [httpx.UnsupportedProtocol("no scheme"), _chat(_GOOD)] + + with pytest.raises(httpx.UnsupportedProtocol): + _evaluate(client, k=2) + + assert client.ask.call_count == 1 + + +def test_a_declined_run_publishes_no_word_count(fake_langfuse: FakeLangfuse) -> None: + client = MagicMock() + client.ask.return_value = ChatResult( + text_response="", raw_response={"description": None, "note": "Object not found", "unusable_retries": 0} + ) + with patch.object(agentic_langfuse, "get_model_version", return_value="gpt-5.2"): + _evaluate( + client, + dataset_item=_dataset_item(expected_output={"expect_note": True}), + langfuse=None, + submit_trace_link=run_trace_link_inline, + ) + + names = {score["name"] for score in _posted_scores(fake_langfuse)} + assert "declined_with_note" in names + assert "word_count" not in names diff --git a/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py b/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py new file mode 100644 index 000000000..86fd302fe --- /dev/null +++ b/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py @@ -0,0 +1,125 @@ +# (C) 2026 GoodData Corporation +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.evaluators.catalog_metadata import CatalogMetadataEvaluator +from gooddata_eval.core.models import CatalogInput, ChatResult, DatasetItem + + +def _make_evaluator() -> CatalogMetadataEvaluator: + with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): + return CatalogMetadataEvaluator() + + +def _item(expected_output: dict, output: str = "description") -> DatasetItem: + return DatasetItem( + id="c1", + dataset_name="d", + test_kind="catalog_metadata", + question="Description for the metric 'ARPU' (Net Sales / Active Customers)", + expected_output=expected_output, + catalog_input=CatalogInput.model_validate({"object_type": "Metric", "object_id": "arpu", "output": output}), + ) + + +def _chat(text: str, note: str | None = None) -> ChatResult: + return ChatResult(text_response=text, raw_response={"note": note, "unusable_retries": 0}) + + +def _judges(ev: CatalogMetadataEvaluator, positive: bool = True, violation: bool = False) -> MagicMock: + """Script both judges; returns the positive judge's mock.""" + positive_score = MagicMock(return_value=(positive, "ok")) + ev._positive_judge.score = positive_score + ev._violation_judge.score = MagicMock(return_value=(violation, "ok")) + return positive_score + + +def test_description_passes_format_and_criteria() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate( + _item({"must_include": ["a"], "must_not_include": ["x"]}), + _chat("Average net sales per active customer. Use it to compare customer value."), + ) + assert res.passed is True + assert res.detail["max_sentences"] is True + assert res.detail["no_ids"] is True + + +@pytest.mark.parametrize( + ("text", "check"), + [ + ("Revenue per customer. " * 30, "max_words"), + ("Uses {metric/net_sales} divided by customers.", "no_ids"), + ("Divides net_sales by active_customers.", "no_ids"), + ("**Average** revenue per customer.", "no_markup"), + ("A. B. C. D. E.", "max_sentences"), + ], +) +def test_description_format_checks_gate(text: str, check: str) -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}), _chat(text)) + assert res.detail[check] is False + assert res.passed is False + + +def test_title_checks() -> None: + ev = _make_evaluator() + _judges(ev) + assert ev.evaluate(_item({}, output="title"), _chat("Average Revenue per Customer")).passed is True + res = ev.evaluate(_item({}, output="title"), _chat("This metric shows average revenue.")) + assert res.detail["no_end_period"] is False + assert res.passed is False + + +def test_format_override_switches_a_check_off() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({"format": {"no_end_period": False}}, output="title"), _chat("Average revenue.")) + assert "no_end_period" not in res.detail + assert res.passed is True + + +def test_empty_text_fails_without_judging() -> None: + ev = _make_evaluator() + positive_score = _judges(ev) + res = ev.evaluate(_item({"must_include": ["a"]}), _chat("")) + assert res.passed is False + assert res.detail["generated"] is False + positive_score.assert_not_called() + + +def test_a_note_next_to_a_text_fails() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}), _chat("Average revenue per customer.", note="Context was partial")) + assert res.detail["no_note"] is False + assert res.passed is False + + +@pytest.mark.parametrize(("text", "note", "passed"), [("", "Not found", True), ("", None, False), ("Text", "x", False)]) +def test_expect_note_decides_alone(text: str, note: str | None, passed: bool) -> None: + ev = _make_evaluator() + positive_score = _judges(ev) + res = ev.evaluate(_item({"expect_note": True, "must_include": ["a"]}), _chat(text, note)) + assert res.passed is passed + positive_score.assert_not_called() + + +def test_violation_fails_the_item() -> None: + ev = _make_evaluator() + _judges(ev, violation=True) + res = ev.evaluate(_item({"must_not_include": ["names a currency"]}), _chat("Price per unit in USD.")) + assert res.detail["exclude_0"] is False + assert res.passed is False + + +@pytest.mark.parametrize( + "expected", + [{"format": {"bold_headline": True}}, {"format": "x"}, {"must_include": "a"}], +) +def test_validate_rejects_bad_expected_output(expected: dict) -> None: + ev = _make_evaluator() + with pytest.raises(ValueError): + ev.evaluate(_item(expected), _chat("x")) diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index ffc7404a8..4265612ff 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -1175,6 +1175,25 @@ def test_cli_routes_takeaway_items_to_takeaway_client() -> None: summary.ask.assert_not_called() +def test_cli_routes_catalog_items_to_the_catalog_client() -> None: + chat, catalog = MagicMock(), MagicMock() + backend = cli_main._RoutingBackend(chat, {"catalog_metadata": catalog}) + raw = { + "id": "lf-cat-1", + "datasetName": "analytics_catalog_generate", + "input": "Title for the metric 'ARPU'", + "metadata": {"catalog_input": {"object_type": "Metric", "object_id": "arpu", "output": "title"}}, + "expectedOutput": {"must_include": ["Average revenue per customer"]}, + } + item = _item_from_raw(raw, dataset_name="analytics_catalog_generate", test_kind="visualization") + + result = backend.ask(item) + + catalog.ask.assert_called_once_with(item) + assert result is catalog.ask.return_value + chat.ask.assert_not_called() + + def test_cli_routes_other_kinds_by_test_kind() -> None: backend, chat, summary, takeaway = _routing_backend() summary_item = DatasetItem( diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py index 028d7d391..2a4ded41f 100644 --- a/packages/gooddata-eval/tests/test_langfuse_source.py +++ b/packages/gooddata-eval/tests/test_langfuse_source.py @@ -323,3 +323,55 @@ def test_a_blank_declaration_does_not_hide_a_real_one_behind_it(): assert _infer_test_kind({"test_kind": "agentic_search"}, "visualization", {"test_kind": "agentic_guardrail"}) == ( "agentic_search" ) + + +def test_langfuse_item_catalog_input_from_metadata_sets_the_kind() -> None: + raw = { + "id": "lf-cat-1", + "datasetName": "ds", + "input": "Title for the metric 'ARPU'", + "metadata": {"catalog_input": {"object_type": "Metric", "object_id": "arpu", "output": "title"}}, + "expectedOutput": {"must_include": ["average revenue"]}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "catalog_metadata" + assert item.catalog_input is not None + assert item.catalog_input.object_id == "arpu" + + +def test_langfuse_item_invalid_catalog_input_raises() -> None: + raw = { + "id": "lf-cat-2", + "datasetName": "ds", + "input": "Title", + "metadata": {"catalog_input": {"object_type": "Metric", "object_id": "arpu", "output": "summary"}}, + "expectedOutput": {}, + } + with pytest.raises(ValueError, match="invalid catalog_input"): + _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + + +def test_langfuse_item_catalog_input_that_is_not_an_object_raises() -> None: + raw = { + "id": "lf-cat-3", + "datasetName": "ds", + "input": "Title", + "metadata": {"catalog_input": "arpu"}, + "expectedOutput": {}, + } + with pytest.raises(ValueError, match="must be a JSON object"): + _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + + +def test_langfuse_item_explicit_test_kind_beats_catalog_input() -> None: + raw = { + "id": "lf-cat-4", + "datasetName": "ds", + "input": "Title", + "metadata": { + "test_kind": "general_question", + "catalog_input": {"object_type": "Metric", "object_id": "arpu", "output": "title"}, + }, + "expectedOutput": {}, + } + assert _item_from_raw(raw, dataset_name="ds", test_kind="visualization").test_kind == "general_question" diff --git a/packages/gooddata-eval/tests/test_models.py b/packages/gooddata-eval/tests/test_models.py index a34cc2a58..2a79507fa 100644 --- a/packages/gooddata-eval/tests/test_models.py +++ b/packages/gooddata-eval/tests/test_models.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation import pytest from gooddata_eval.core.models import ( + CatalogInput, ChatResult, CreatedVisualization, DatasetItem, @@ -221,3 +222,10 @@ def test_dataset_item_takeaway_input_defaults_to_none() -> None: {"id": "q1", "dataset_name": "d1", "test_kind": "dashboard_summary", "question": "q", "expected_output": {}} ) assert item.takeaway_input is None + + +def test_catalog_input_rejects_unknown_keys_and_types() -> None: + with pytest.raises(ValueError): + CatalogInput.model_validate({"object_type": "Metric", "object_id": "a", "output": "title", "x": 1}) + with pytest.raises(ValueError): + CatalogInput.model_validate({"object_type": "Parameter", "object_id": "a", "output": "title"}) diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index d233f0574..5058112c0 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -135,6 +135,7 @@ def test_run_items_routes_all_supported_kinds(): "dashboard_summary", "knowledge_question", "document_takeaway", + "catalog_metadata", } assert expected_kinds == supported_test_kinds() From 4ef1c9e3bd62849aaa9ed3a20609887d7d7a4c03 Mon Sep 17 00:00:00 2001 From: duongthimy Date: Tue, 17 Nov 2026 13:57:09 +0100 Subject: [PATCH 2/3] fix(gooddata-eval): tighten catalog_metadata checks from review - blank text and blank note count as no answer, so they are asked again and never pass as a decline - no_markup also catches single emphasis (*x*, _x_) - format overrides must be a positive integer (limits) or true, false or null; checked before any request - calibration counts an ungraded judge run as a mismatch jira: QA-30180 risk: low --- .../calibrate_catalog_metadata_judge.py | 11 ++++++++--- .../gooddata_eval/core/catalog/http_client.py | 17 +++++++++-------- .../core/evaluators/catalog_metadata.py | 12 +++++++++++- .../tests/test_catalog_metadata_client.py | 6 +++++- .../tests/test_catalog_metadata_evaluator.py | 19 ++++++++++++++++++- 5 files changed, 51 insertions(+), 14 deletions(-) diff --git a/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py b/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py index 7d96b7255..03d553d50 100644 --- a/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py +++ b/packages/gooddata-eval/scripts/calibrate_catalog_metadata_judge.py @@ -49,11 +49,16 @@ def main() -> int: ) chat = ChatResult(text_response=case["text"], raw_response={"note": None}) key = f"{case['kind']}_0" - satisfied = sum(evaluator.evaluate(item, chat).detail.get(key) is True for _ in range(RUNS)) + verdicts = [evaluator.evaluate(item, chat).detail.get(key) for _ in range(RUNS)] + ungraded = sum(not isinstance(v, bool) for v in verdicts) + satisfied = sum(v is True for v in verdicts) want = RUNS if case["want"] == "pass" else 0 - ok = satisfied == want + # An ungraded run proves nothing, on either side. + ok = satisfied == want and not ungraded mismatches += not ok - print(f"{case['label']:48} want {case['want']:4} satisfied {satisfied}/{RUNS} {'OK' if ok else 'MISMATCH'}") + print( + f"{case['label']:48} want {case['want']:4} satisfied {satisfied}/{RUNS}{f' ungraded {ungraded}' if ungraded else ''} {'OK' if ok else 'MISMATCH'}" + ) print(f"\n{'all calibration texts on the right side' if not mismatches else f'{mismatches} mismatch(es)'}") return 1 if mismatches else 0 diff --git a/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py b/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py index f6ff13dfa..3ea2b7761 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/catalog/http_client.py @@ -29,13 +29,15 @@ _PLACEHOLDER = "string" +def _text(value: Any) -> str: + """A field's text for the usable check: blank and the schema placeholder count as none.""" + text = value.strip() if isinstance(value, str) else "" + return "" if text == _PLACEHOLDER else text + + def is_unusable(data: dict[str, Any], output: str) -> bool: """Whether a 200 answer carries neither a generated text nor a reason for its absence.""" - value = data.get(output) - note = data.get("note") - if value == _PLACEHOLDER: - return True - return not value and not note + return not _text(data.get(output)) and not _text(data.get("note")) class CatalogMetadataClient: @@ -84,10 +86,9 @@ def _post() -> httpx.Response: while is_unusable(data, ci.output) and retries < self._unusable_retries: retries += 1 data = retry_transient(_post, is_retryable=is_retryable_request_error).json() - value = data.get(ci.output) - text = value if isinstance(value, str) and value != _PLACEHOLDER else "" + text = _text(data.get(ci.output)) # The placeholder is no reason either: a declined answer must carry a real note. - note = data.get("note") if data.get("note") != _PLACEHOLDER else None + note = _text(data.get("note")) or None return ChatResult( text_response=text, raw_response={**data, "note": note, "output": ci.output, "unusable_retries": retries}, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py index f74c7db09..0853dbb37 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py @@ -55,7 +55,12 @@ "Score 0 if it does not exhibit it.", ] -_MARKUP = re.compile(r"\*\*|`|^\s*#{1,6}\s|^\s*(?:[-*•]|\d+[.)])\s", re.MULTILINE) +# Bold, single emphasis (*x*, _x_), code, a heading or a list item. +_MARKUP = re.compile( + r"\*\*|`|(? None: raise ValueError( f"catalog_metadata item '{item.id}': unknown format key(s) {unknown}; known: {sorted(known)}" ) + for key, value in overrides.items(): + is_limit = isinstance(value, int) and not isinstance(value, bool) and value > 0 + if not (value is None or value is False or (is_limit if key in _LIMITS else value is True)): + kind = "a positive integer" if key in _LIMITS else "true" + raise ValueError(f"catalog_metadata item '{item.id}': format.{key} must be {kind}, false or null") for key in ("must_include", "must_not_include", "rubric"): if not isinstance(expected.get(key, []), list): raise ValueError(f"catalog_metadata item '{item.id}': {key} must be a JSON array") diff --git a/packages/gooddata-eval/tests/test_catalog_metadata_client.py b/packages/gooddata-eval/tests/test_catalog_metadata_client.py index 8cb4f868e..4359b41a5 100644 --- a/packages/gooddata-eval/tests/test_catalog_metadata_client.py +++ b/packages/gooddata-eval/tests/test_catalog_metadata_client.py @@ -56,7 +56,10 @@ def test_posts_object_to_the_action_of_the_output(output: str, action: str) -> N } -@pytest.mark.parametrize("unusable", [{"title": None, "note": None}, {"title": "string", "note": "string"}]) +@pytest.mark.parametrize( + "unusable", + [{"title": None, "note": None}, {"title": "string", "note": "string"}, {"title": " ", "note": " "}], +) def test_asks_again_once_after_an_unusable_answer(unusable: dict) -> None: captured: list = [] client = _client(_sequence(captured, [unusable, {"title": "Average Revenue", "note": None}])) @@ -108,6 +111,7 @@ def test_is_unusable() -> None: assert is_unusable({"title": "string", "note": None}, "title") assert not is_unusable({"title": None, "note": "Not found"}, "title") assert not is_unusable({"title": "Revenue", "note": None}, "title") + assert is_unusable({"title": " ", "note": " "}, "title") def test_a_placeholder_note_is_dropped() -> None: diff --git a/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py b/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py index 86fd302fe..d733d470b 100644 --- a/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py +++ b/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py @@ -53,6 +53,8 @@ def test_description_passes_format_and_criteria() -> None: ("Uses {metric/net_sales} divided by customers.", "no_ids"), ("Divides net_sales by active_customers.", "no_ids"), ("**Average** revenue per customer.", "no_markup"), + ("The *average* revenue per customer.", "no_markup"), + ("The _average_ revenue per customer.", "no_markup"), ("A. B. C. D. E.", "max_sentences"), ], ) @@ -117,9 +119,24 @@ def test_violation_fails_the_item() -> None: @pytest.mark.parametrize( "expected", - [{"format": {"bold_headline": True}}, {"format": "x"}, {"must_include": "a"}], + [ + {"format": {"bold_headline": True}}, + {"format": "x"}, + {"must_include": "a"}, + {"format": {"max_words": "10"}}, + {"format": {"max_words": True}}, + {"format": {"no_markup": "yes"}}, + ], ) def test_validate_rejects_bad_expected_output(expected: dict) -> None: ev = _make_evaluator() with pytest.raises(ValueError): ev.evaluate(_item(expected), _chat("x")) + + +def test_snake_case_is_an_id_not_emphasis() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}), _chat("Uses net_sales over 2 * 3 periods.")) + assert res.detail["no_markup"] is True + assert res.detail["no_ids"] is False From 27bc43beaa8c97a074a105c63171b0d4b4644c05 Mon Sep 17 00:00:00 2001 From: duongthimy Date: Wed, 18 Nov 2026 14:03:51 +0100 Subject: [PATCH 3/3] fix(gooddata-eval): keep catalog titles like "# of Orders" passing - no_markup: a leading "# of" is a metric name, not a heading - word and sentence counts move to _text_utils; each evaluator passes its own abbreviation list, so catalog keeps "Avg.", "Est.", "U.K." in one sentence and takeaway verdicts stay as they are jira: QA-30180 risk: low --- .../core/evaluators/_text_utils.py | 35 ++++++++++++++++- .../core/evaluators/catalog_metadata.py | 38 ++++++++++++++++--- .../gooddata_eval/core/evaluators/takeaway.py | 24 ++---------- .../tests/test_catalog_metadata_evaluator.py | 17 +++++++++ 4 files changed, 88 insertions(+), 26 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_text_utils.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_text_utils.py index 356a829fa..e4343f7e7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_text_utils.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_text_utils.py @@ -1,5 +1,8 @@ # (C) 2026 GoodData Corporation -"""Shared text-extraction helpers for text-answer evaluators.""" +"""Shared text helpers for text-answer evaluators: answer extraction, word and sentence counts.""" + +import re +from collections.abc import Collection from gooddata_eval.core.chat.render import render_answer_text from gooddata_eval.core.models import ChatResult @@ -8,3 +11,33 @@ def extract_text(chat_result: ChatResult) -> str: """Extract the agent's answer: prose plus any content-bearing non-text part.""" return render_answer_text(chat_result) + + +# A terminator, any closing quotes or brackets, then whitespace before the next sentence. +_SENTENCE_BREAK = re.compile(r"[.!?]+[\"'”’»)\]]*\s+(?=\S)") +_SENTENCE_OPENERS = "\"'“‘„«([" + + +def count_words(text: str) -> int: + """Words in `text`: whitespace-separated tokens holding a letter or digit, bold markers ignored.""" + return sum(1 for token in text.replace("**", " ").split() if any(ch.isalnum() for ch in token)) + + +def count_sentences(text: str, abbreviations: Collection[str]) -> int: + """Sentences in `text`; a break needs the next sentence to open with a capital, digit or quote. + + That keeps "vs. last" or "např. v" from splitting one sentence in two. A terminator right + after a token in `abbreviations` (lowercase, without its final dot: "vs", "e.g") is no + break either. Each evaluator passes its own list, so tuning one cannot move another's verdicts. + """ + plain = text.replace("**", "").strip() + if not plain: + return 0 + breaks = 0 + for match in _SENTENCE_BREAK.finditer(plain): + nxt = plain[match.end()] + token = (plain[: match.start()].rsplit(maxsplit=1) or [""])[-1].lstrip("\"'“‘„«([").lower() + if token in abbreviations: + continue + breaks += nxt.isupper() or nxt.isdigit() or nxt in _SENTENCE_OPENERS + return breaks + 1 diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py index 0853dbb37..ca93208c9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/catalog_metadata.py @@ -27,8 +27,8 @@ from gooddata_eval.core.evaluators._llm_judge import LLMJudge from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators._text_utils import count_sentences, count_words from gooddata_eval.core.evaluators.base import ItemEvaluation -from gooddata_eval.core.evaluators.takeaway import _sentences, _words from gooddata_eval.core.models import ChatResult, DatasetItem _DEFAULT_FORMAT: dict[str, dict[str, Any]] = { @@ -55,12 +55,40 @@ "Score 0 if it does not exhibit it.", ] -# Bold, single emphasis (*x*, _x_), code, a heading or a list item. +# Bold, single emphasis (*x*, _x_), code, a heading or a list item. A leading "# of" is a metric +# name ("# of Orders"), not a heading. _MARKUP = re.compile( - r"\*\*|`|(? ItemEvaluation def _format_checks(output: str, expected: dict[str, Any], text: str) -> dict[str, Any]: """Deterministic checks keyed by name, plus `word_count`.""" fmt = {**_DEFAULT_FORMAT[output], **(expected.get("format") or {})} - checks: dict[str, Any] = {"word_count": _words(text)} + checks: dict[str, Any] = {"word_count": count_words(text)} if _limit_on(fmt["max_words"]): checks["max_words"] = checks["word_count"] <= fmt["max_words"] if _limit_on(fmt["max_sentences"]): - checks["max_sentences"] = _sentences(text) <= fmt["max_sentences"] + checks["max_sentences"] = count_sentences(text, _ABBREVIATIONS) <= fmt["max_sentences"] if fmt["no_end_period"]: checks["no_end_period"] = not text.endswith(".") if fmt["no_markup"]: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py index a92ccbfe1..ccc6ef01b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py @@ -30,6 +30,7 @@ from gooddata_eval.core.evaluators._llm_judge import LLMJudge from gooddata_eval.core.evaluators._rubric import Criterion, grade_rubric +from gooddata_eval.core.evaluators._text_utils import count_sentences, count_words from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem @@ -97,9 +98,6 @@ ] _HEADLINE = re.compile(r"^\s*\*\*(.+?)\*\*", re.DOTALL) -# A terminator, any closing quotes or brackets, then whitespace before the next sentence. -_SENTENCE_BREAK = re.compile(r"[.!?]+[\"'”’»)\]]*\s+(?=\S)") -_SENTENCE_OPENERS = "\"'“‘„«([" _ABBREVIATIONS = frozenset({"vs", "e.g", "i.e", "u.s", "cca", "např", "tj", "tzn", "apod", "resp", "atd"}) _BLANK_LINE = re.compile(r"\n\s*\n") _HEADING_LINE = re.compile(r"^\s*#{1,6}\s", re.MULTILINE) @@ -115,7 +113,7 @@ def _words(text: str) -> int: - return sum(1 for token in text.replace("**", " ").split() if any(ch.isalnum() for ch in token)) + return count_words(text) def _limit_on(value: Any) -> bool: @@ -123,22 +121,8 @@ def _limit_on(value: Any) -> bool: def _sentences(text: str) -> int: - """Sentences in `text`; a break needs the next sentence to open with a capital, digit or quote. - - That keeps "vs. last", "U.S. sales" or "např. v" from splitting one sentence in two; a - terminator right after a known abbreviation ("vs. Europe", "e.g. Nordics") is no break. - """ - plain = text.replace("**", "").strip() - if not plain: - return 0 - breaks = 0 - for match in _SENTENCE_BREAK.finditer(plain): - nxt = plain[match.end()] - token = (plain[: match.start()].rsplit(maxsplit=1) or [""])[-1].lstrip("\"'“‘„«([").lower() - if token in _ABBREVIATIONS: - continue - breaks += nxt.isupper() or nxt.isdigit() or nxt in _SENTENCE_OPENERS - return breaks + 1 + """Sentences in `text`, with the takeaway's abbreviations ("vs. Europe", "e.g. Nordics").""" + return count_sentences(text, _ABBREVIATIONS) def _has_meta_opener(text: str, headline: str | None, body: str) -> bool: diff --git a/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py b/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py index d733d470b..7ee5dde07 100644 --- a/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py +++ b/packages/gooddata-eval/tests/test_catalog_metadata_evaluator.py @@ -140,3 +140,20 @@ def test_snake_case_is_an_id_not_emphasis() -> None: res = ev.evaluate(_item({}), _chat("Uses net_sales over 2 * 3 periods.")) assert res.detail["no_markup"] is True assert res.detail["no_ids"] is False + + +@pytest.mark.parametrize("title", ["# of Orders", "Avg. Order Value by Region", "Est. Revenue", "U.K. Sales"]) +def test_title_keeps_metric_names_and_label_abbreviations(title: str) -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}, output="title"), _chat(title)) + assert res.detail["no_markup"] is True + assert res.detail["max_sentences"] is True + assert res.passed is True + + +def test_a_real_heading_is_still_markup() -> None: + ev = _make_evaluator() + _judges(ev) + res = ev.evaluate(_item({}), _chat("# Revenue\nShows revenue by region.")) + assert res.detail["no_markup"] is False