diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfda520f4..8d323ce40 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -852,7 +852,8 @@ an item carrying `takeaway_input` is a `document_takeaway` item unless it declar `word_count` and `headline_word_count` are reported in the item detail. The default judge criteria are calibrated with `make calibrate-takeaway` (needs `OPENAI_API_KEY`); re-run it -after changing their wording. +after changing their wording. The judge reads the item's `input` as the page label, and the +label can change a verdict, so describe the page and the request in plain words. The client retries 429/5xx answers and connection failures. A request error that survives the retries is handled differently by the two entry points: `gd-eval run` aborts the item diff --git a/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py b/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py index ce7630650..f03f3da3c 100644 --- a/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py +++ b/packages/gooddata-eval/scripts/calibrate_takeaway_judge.py @@ -6,7 +6,9 @@ tests/fixtures/takeaway_calibration.json: - a "pass" text must satisfy every default judge criterion on every run; -- a "fail" text must violate each criterion it names on every run. +- a "fail" text must violate each criterion it names on every run; +- a text's own "question" is the page label the judge reads (default: the fixture's). The label + can change the verdict; a text from a real run usually keeps its item's label. Run it after changing any DEFAULT_MUST_INCLUDE / DEFAULT_MUST_NOT_INCLUDE wording: @@ -47,15 +49,18 @@ def main() -> int: id=case["label"], dataset_name="takeaway_calibration", test_kind="document_takeaway", - question=fixture["question"], + question=case.get("question", fixture["question"]), expected_output={}, takeaway_input=TakeawayInput(visualizations=["calibration"]), ) chat = ChatResult(text_response=case["text"], raw_response={"takeaway": case["text"], "noData": False}) satisfied = dict.fromkeys(CRITERIA.values(), 0) + # An ungraded criterion has no key in detail; never count it as a verdict. + ungraded = dict.fromkeys(CRITERIA.values(), 0) for _ in range(RUNS): detail = evaluator.evaluate(item, chat).detail for key, name in CRITERIA.items(): + ungraded[name] += key not in detail satisfied[name] += detail.get(key) is True want_pass = case["want"] == "pass" must = dict.fromkeys(satisfied, RUNS) if want_pass else dict.fromkeys(case["criteria"], 0) @@ -63,10 +68,11 @@ def main() -> int: for name, count in satisfied.items(): flag = "" if name in must: - ok = count == must[name] + ok = count == must[name] and not ungraded[name] mismatches += not ok flag = "OK" if ok else "MISMATCH" - print(f" {name:20} satisfied {count}/{RUNS} {flag}") + note = f" ({ungraded[name]} ungraded)" if ungraded[name] else "" + print(f" {name:20} satisfied {count}/{RUNS}{note} {flag}") print(f"\n{'all calibration texts on the right side' if not mismatches else f'{mismatches} mismatch(es)'}") return 1 if mismatches else 0 diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py index 494910938..a92ccbfe1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/takeaway.py @@ -45,12 +45,16 @@ "'Key takeaways' or 'Things to watch', or lists several separate findings one after another, " "or gives recommendations, an outlook or a closing conclusion." ), - "chart_walkthrough": "Walks through the charts one by one instead of stating one finding that connects them.", + "chart_walkthrough": ( + "Walks through two or more charts one by one instead of stating one finding that connects them." + ), "background_opener": "Opens with background or a description of the page instead of the finding itself.", "exact_figure": ( "States a figure in digits: an amount, count, percentage or share, including rounded or hedged ones " - "such as 'about $270M', 'roughly 1.2K', 'in the low $100Ks' or '~40%'. A rough size or direction in words, such as " - "'roughly doubled', 'about a third' or 'most orders', does not count. Numbers that are part of a " + "such as 'about $270M', 'roughly 1.2K', 'in the low $100Ks' or '~40%'. Only digits count: a size, " + "range or direction written in words, in any language, does not count, even when hedged, such as " + "'roughly doubled', 'about a third', 'nearly nine-tenths', 'roughly a million', 'the low teens', " + "'a six- to seven-figure range', 'most orders' or 'etwa eine Million'. Numbers that are part of a " "visualization or metric title, such as 'Top 10', do not count." ), "date_or_period": ( @@ -61,7 +65,7 @@ "Makes an explicit causal claim (such as 'because', 'due to' or 'driven by') that attributes a change " "to a factor that is not in the data, such as a campaign, a season or a market event. Comparisons " "between segments, and naming the segment in the data that led a change (such as 'led by the " - "Nordics'), are not causal claims." + "Nordics' or 'driven by new customers'), are not causal claims." ), "chart_description": "Describes what a chart displays instead of what the data means.", } diff --git a/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json b/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json index 6ef3502de..f6a240deb 100644 --- a/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json +++ b/packages/gooddata-eval/tests/fixtures/takeaway_calibration.json @@ -26,6 +26,53 @@ "want": "pass", "text": "**Audio and video now carry the store.** They bring in about a third of sales, and their sales roughly doubled compared with last period." }, + { + "label": "number words: roughly a million (real output)", + "want": "pass", + "question": "Customers page, slot asks for English, document instruction asks for German", + "text": "**New customers lead the period**\nNew customer activity is substantially higher than returning activity in the active-customer breakdown, while the top spenders each contribute roughly a million in net sales." + }, + { + "label": "number words: in the low teens", + "want": "pass", + "question": "Customers page, slot asks to focus on returning customers", + "text": "**Returning customers stay a small group**\nReturning customers stay in the low teens on most days, while new customers make up nearly all of the activity." + }, + { + "label": "number words in German", + "want": "pass", + "question": "Customers page, document instruction asks for German", + "text": "**Neukunden tragen das Wachstum**\nFast alle aktiven Kunden sind neu, und die Top-Kunden bringen jeweils Umsätze im Millionenbereich." + }, + { + "label": "number words in German (real output)", + "want": "pass", + "question": "Customers page, document instruction asks for German", + "text": "**Neukunden prägen die Aktivität**\nDie aktiven Kunden sind ausschließlich Neukunden. Die zehn umsatzstärksten Kunden liegen jeweils im Bereich von etwa einer Million Net Sales." + }, + { + "label": "fraction in words", + "want": "pass", + "text": "**Three categories carry the store**\nAudio & Video, Computers and Furniture bring in nearly nine-tenths of sales, leaving little for the rest." + }, + { + "label": "magnitude range in German words (real output)", + "want": "pass", + "question": "Customers page, document instruction asks for German", + "text": "**Neukunden tragen die Aktivität**\nIm ausgewählten Zeitraum bestehen die aktiven Kunden vollständig aus Neukunden; Rückkehrer sind nicht erfasst. Die umsatzstärksten Kunden liegen jeweils im hohen sechs- bis niedrigen siebenstelligen Bereich." + }, + { + "label": "one chart, one finding", + "want": "pass", + "question": "Single Revenue Trend chart, no instructions", + "text": "**Net sales climb steadily**\nNet sales rise through most of the period, with only short dips along the way." + }, + { + "label": "segment in the data named as the driver", + "want": "pass", + "question": "Customers page with a trend, a top-10 list and a breakdown, no instructions", + "text": "**New customers carry the growth**\nActive customer growth is driven entirely by new customers, while returning customers stay flat." + }, { "label": "executive summary", "want": "fail", @@ -63,6 +110,48 @@ "want": "fail", "criteria": ["unshown_cause"], "text": "**Sales jumped in the West.** The rise came because the summer campaign drew in new buyers." + }, + { + "label": "hedged figure in digits (real output)", + "want": "fail", + "question": "Customers page, slot asks for English, document instruction asks for German", + "criteria": ["exact_figure"], + "text": "**Growth is acquisition-led**\nNew Customers make up the active base, while the leading spenders contribute roughly $1M each. Return Customers are absent from the breakdown, pointing to limited repeat engagement in this view." + }, + { + "label": "percentage change in digits", + "want": "fail", + "question": "Customers page, slot asks to focus on returning customers", + "criteria": ["exact_figure"], + "text": "**Returning customers spend more**\nReturning customers spend roughly 25% higher per order than new customers, though they place fewer orders." + }, + { + "label": "walk-through under a finding headline", + "want": "fail", + "question": "Sales page with three charts, no instructions", + "criteria": ["chart_walkthrough"], + "text": "**Growth comes from new customers.** The trend chart has sales rising, the category chart has Audio & Video in front, and the order breakdown has most orders processed." + }, + { + "label": "outside cause after 'driven by'", + "want": "fail", + "question": "Sales page with three charts, no instructions", + "criteria": ["unshown_cause"], + "text": "**Sales peaked late in the period**\nThe late rise was driven by the holiday season, while the other months stayed flat." + }, + { + "label": "figure in digits in German", + "want": "fail", + "question": "Customers page, document instruction asks for German", + "criteria": ["exact_figure"], + "text": "**Neukunden tragen das Wachstum**\nNeukunden machen rund 40 % der aktiven Kunden aus, und die Top-Kunden bringen jeweils etwa 1 Mio. € Umsatz." + }, + { + "label": "outside cause after 'driven by' (marketing)", + "want": "fail", + "question": "Customers page with a trend, a top-10 list and a breakdown, no instructions", + "criteria": ["unshown_cause"], + "text": "**Growth is driven by marketing**\nNew customers rose because of stronger marketing, while returning customers stayed flat." } ] } diff --git a/packages/gooddata-eval/tests/test_takeaway_evaluator.py b/packages/gooddata-eval/tests/test_takeaway_evaluator.py index 26a93d780..84e0c227e 100644 --- a/packages/gooddata-eval/tests/test_takeaway_evaluator.py +++ b/packages/gooddata-eval/tests/test_takeaway_evaluator.py @@ -360,6 +360,9 @@ def test_title_number_criterion_text_present() -> None: assert "'about $270M'" in criterion assert "'about a third'" in criterion assert "'roughly doubled'" in criterion + assert "'roughly a million'" in criterion + assert "Only digits count" in criterion + assert "'nearly nine-tenths'" in criterion def test_calibrated_criterion_wording() -> None: @@ -374,6 +377,8 @@ def test_calibrated_criterion_wording() -> None: } assert "Things to watch" in DEFAULT_MUST_NOT_INCLUDE["executive_summary"] assert "led by" in DEFAULT_MUST_NOT_INCLUDE["unshown_cause"] + assert "'driven by new customers'" in DEFAULT_MUST_NOT_INCLUDE["unshown_cause"] + assert "two or more charts" in DEFAULT_MUST_NOT_INCLUDE["chart_walkthrough"] assert set(DEFAULT_MUST_INCLUDE) == {"one_finding"}