Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/gooddata-eval/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -852,7 +852,8 @@ an item carrying `takeaway_input` is a `document_takeaway` item unless it declar

`word_count` and `headline_word_count` are reported in the item detail. The default judge
criteria are calibrated with `make calibrate-takeaway` (needs `OPENAI_API_KEY`); re-run it
after changing their wording.
after changing their wording. The judge reads the item's `input` as the page label, and the
label can change a verdict, so describe the page and the request in plain words.

The client retries 429/5xx answers and connection failures. A request error that survives
the retries is handled differently by the two entry points: `gd-eval run` aborts the item
Expand Down
14 changes: 10 additions & 4 deletions packages/gooddata-eval/scripts/calibrate_takeaway_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
tests/fixtures/takeaway_calibration.json:

- a "pass" text must satisfy every default judge criterion on every run;
- a "fail" text must violate each criterion it names on every run.
- a "fail" text must violate each criterion it names on every run;
- a text's own "question" is the page label the judge reads (default: the fixture's). The label
can change the verdict; a text from a real run usually keeps its item's label.

Run it after changing any DEFAULT_MUST_INCLUDE / DEFAULT_MUST_NOT_INCLUDE wording:

Expand Down Expand Up @@ -47,26 +49,30 @@ def main() -> int:
id=case["label"],
dataset_name="takeaway_calibration",
test_kind="document_takeaway",
question=fixture["question"],
question=case.get("question", fixture["question"]),
expected_output={},
takeaway_input=TakeawayInput(visualizations=["calibration"]),
)
chat = ChatResult(text_response=case["text"], raw_response={"takeaway": case["text"], "noData": False})
satisfied = dict.fromkeys(CRITERIA.values(), 0)
# An ungraded criterion has no key in detail; never count it as a verdict.
ungraded = dict.fromkeys(CRITERIA.values(), 0)
for _ in range(RUNS):
detail = evaluator.evaluate(item, chat).detail
for key, name in CRITERIA.items():
ungraded[name] += key not in detail
satisfied[name] += detail.get(key) is True
want_pass = case["want"] == "pass"
must = dict.fromkeys(satisfied, RUNS) if want_pass else dict.fromkeys(case["criteria"], 0)
print(f"\n{case['label']} (want {case['want']})")
for name, count in satisfied.items():
flag = ""
if name in must:
ok = count == must[name]
ok = count == must[name] and not ungraded[name]
mismatches += not ok
flag = "OK" if ok else "MISMATCH"
print(f" {name:20} satisfied {count}/{RUNS} {flag}")
note = f" ({ungraded[name]} ungraded)" if ungraded[name] else ""
print(f" {name:20} satisfied {count}/{RUNS}{note} {flag}")
print(f"\n{'all calibration texts on the right side' if not mismatches else f'{mismatches} mismatch(es)'}")
return 1 if mismatches else 0

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,16 @@
"'Key takeaways' or 'Things to watch', or lists several separate findings one after another, "
"or gives recommendations, an outlook or a closing conclusion."
),
"chart_walkthrough": "Walks through the charts one by one instead of stating one finding that connects them.",
"chart_walkthrough": (
"Walks through two or more charts one by one instead of stating one finding that connects them."
),
"background_opener": "Opens with background or a description of the page instead of the finding itself.",
"exact_figure": (
"States a figure in digits: an amount, count, percentage or share, including rounded or hedged ones "
"such as 'about $270M', 'roughly 1.2K', 'in the low $100Ks' or '~40%'. A rough size or direction in words, such as "
"'roughly doubled', 'about a third' or 'most orders', does not count. Numbers that are part of a "
"such as 'about $270M', 'roughly 1.2K', 'in the low $100Ks' or '~40%'. Only digits count: a size, "
"range or direction written in words, in any language, does not count, even when hedged, such as "
"'roughly doubled', 'about a third', 'nearly nine-tenths', 'roughly a million', 'the low teens', "
"'a six- to seven-figure range', 'most orders' or 'etwa eine Million'. Numbers that are part of a "
"visualization or metric title, such as 'Top 10', do not count."
),
"date_or_period": (
Expand All @@ -61,7 +65,7 @@
"Makes an explicit causal claim (such as 'because', 'due to' or 'driven by') that attributes a change "
"to a factor that is not in the data, such as a campaign, a season or a market event. Comparisons "
"between segments, and naming the segment in the data that led a change (such as 'led by the "
"Nordics'), are not causal claims."
"Nordics' or 'driven by new customers'), are not causal claims."
),
"chart_description": "Describes what a chart displays instead of what the data means.",
}
Expand Down
89 changes: 89 additions & 0 deletions packages/gooddata-eval/tests/fixtures/takeaway_calibration.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,53 @@
"want": "pass",
"text": "**Audio and video now carry the store.** They bring in about a third of sales, and their sales roughly doubled compared with last period."
},
{
"label": "number words: roughly a million (real output)",
"want": "pass",
"question": "Customers page, slot asks for English, document instruction asks for German",
"text": "**New customers lead the period**\nNew customer activity is substantially higher than returning activity in the active-customer breakdown, while the top spenders each contribute roughly a million in net sales."
},
{
"label": "number words: in the low teens",
"want": "pass",
"question": "Customers page, slot asks to focus on returning customers",
"text": "**Returning customers stay a small group**\nReturning customers stay in the low teens on most days, while new customers make up nearly all of the activity."
},
{
"label": "number words in German",
"want": "pass",
"question": "Customers page, document instruction asks for German",
"text": "**Neukunden tragen das Wachstum**\nFast alle aktiven Kunden sind neu, und die Top-Kunden bringen jeweils Umsätze im Millionenbereich."
},
{
"label": "number words in German (real output)",
"want": "pass",
"question": "Customers page, document instruction asks for German",
"text": "**Neukunden prägen die Aktivität**\nDie aktiven Kunden sind ausschließlich Neukunden. Die zehn umsatzstärksten Kunden liegen jeweils im Bereich von etwa einer Million Net Sales."
},
{
"label": "fraction in words",
"want": "pass",
"text": "**Three categories carry the store**\nAudio & Video, Computers and Furniture bring in nearly nine-tenths of sales, leaving little for the rest."
},
{
"label": "magnitude range in German words (real output)",
"want": "pass",
"question": "Customers page, document instruction asks for German",
"text": "**Neukunden tragen die Aktivität**\nIm ausgewählten Zeitraum bestehen die aktiven Kunden vollständig aus Neukunden; Rückkehrer sind nicht erfasst. Die umsatzstärksten Kunden liegen jeweils im hohen sechs- bis niedrigen siebenstelligen Bereich."
},
{
"label": "one chart, one finding",
"want": "pass",
"question": "Single Revenue Trend chart, no instructions",
"text": "**Net sales climb steadily**\nNet sales rise through most of the period, with only short dips along the way."
},
{
"label": "segment in the data named as the driver",
"want": "pass",
"question": "Customers page with a trend, a top-10 list and a breakdown, no instructions",
"text": "**New customers carry the growth**\nActive customer growth is driven entirely by new customers, while returning customers stay flat."
},
{
"label": "executive summary",
"want": "fail",
Expand Down Expand Up @@ -63,6 +110,48 @@
"want": "fail",
"criteria": ["unshown_cause"],
"text": "**Sales jumped in the West.** The rise came because the summer campaign drew in new buyers."
},
{
"label": "hedged figure in digits (real output)",
"want": "fail",
"question": "Customers page, slot asks for English, document instruction asks for German",
"criteria": ["exact_figure"],
"text": "**Growth is acquisition-led**\nNew Customers make up the active base, while the leading spenders contribute roughly $1M each. Return Customers are absent from the breakdown, pointing to limited repeat engagement in this view."
},
{
"label": "percentage change in digits",
"want": "fail",
"question": "Customers page, slot asks to focus on returning customers",
"criteria": ["exact_figure"],
"text": "**Returning customers spend more**\nReturning customers spend roughly 25% higher per order than new customers, though they place fewer orders."
},
{
"label": "walk-through under a finding headline",
"want": "fail",
"question": "Sales page with three charts, no instructions",
"criteria": ["chart_walkthrough"],
"text": "**Growth comes from new customers.** The trend chart has sales rising, the category chart has Audio & Video in front, and the order breakdown has most orders processed."
},
{
"label": "outside cause after 'driven by'",
"want": "fail",
"question": "Sales page with three charts, no instructions",
"criteria": ["unshown_cause"],
"text": "**Sales peaked late in the period**\nThe late rise was driven by the holiday season, while the other months stayed flat."
},
{
"label": "figure in digits in German",
"want": "fail",
"question": "Customers page, document instruction asks for German",
"criteria": ["exact_figure"],
"text": "**Neukunden tragen das Wachstum**\nNeukunden machen rund 40 % der aktiven Kunden aus, und die Top-Kunden bringen jeweils etwa 1 Mio. € Umsatz."
},
{
"label": "outside cause after 'driven by' (marketing)",
"want": "fail",
"question": "Customers page with a trend, a top-10 list and a breakdown, no instructions",
"criteria": ["unshown_cause"],
"text": "**Growth is driven by marketing**\nNew customers rose because of stronger marketing, while returning customers stayed flat."
}
]
}
5 changes: 5 additions & 0 deletions packages/gooddata-eval/tests/test_takeaway_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,9 @@ def test_title_number_criterion_text_present() -> None:
assert "'about $270M'" in criterion
assert "'about a third'" in criterion
assert "'roughly doubled'" in criterion
assert "'roughly a million'" in criterion
assert "Only digits count" in criterion
assert "'nearly nine-tenths'" in criterion


def test_calibrated_criterion_wording() -> None:
Expand All @@ -374,6 +377,8 @@ def test_calibrated_criterion_wording() -> None:
}
assert "Things to watch" in DEFAULT_MUST_NOT_INCLUDE["executive_summary"]
assert "led by" in DEFAULT_MUST_NOT_INCLUDE["unshown_cause"]
assert "'driven by new customers'" in DEFAULT_MUST_NOT_INCLUDE["unshown_cause"]
assert "two or more charts" in DEFAULT_MUST_NOT_INCLUDE["chart_walkthrough"]
assert set(DEFAULT_MUST_INCLUDE) == {"one_finding"}


Expand Down
Loading