From 9be7fa50e3a41106279072f74f98e37f2bac8483 Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Wed, 18 Nov 2026 15:51:48 +0700 Subject: [PATCH] fix(gooddata-eval): sim user prompt names the date dataset and pick When the agent asked which of several objects to use, the visualization and alert simulated users sometimes named the one the fixture does not expect, and the agent built what it was told: Order date vs Return date, new_customers vs new_customers_monthly, of_orders vs orders_all_time. Visualization: the goal's date filter gave only its period, never its date dataset, so a choice between Order date and Return date had no answer in the prompt. It now says "date filter on the return date". The prompt also tells the simulated user to choose the alternative that matches its goal and repeat it as the agent wrote it, and that a date attribute such as order_date.year belongs to the order date. Objects stay named by title: giving them by reference made the simulated user volunteer references to the agent. Alert: the goal has no filters, and rule 5 says "no filters ... all time", which matched an offered metric described as ignoring filters and counting all orders. The prompt now tells the simulated user to choose exactly the goal's metric and that such a metric is a different one. Replayed the three recorded agent messages 20 times each against the real models: all correct, and no reference volunteered to a chart type, open or period question. Models, scoring and gates are unchanged. jira: QA-29478 risk: low Co-Authored-By: Claude Opus 5.5 --- .../gooddata_eval/core/agentic/alert_skill.py | 22 +++++++- .../core/agentic/visualization.py | 19 +++++-- .../tests/test_agentic_alert_skill.py | 24 +++++++++ .../tests/test_agentic_visualization.py | 52 +++++++++++++++++++ 4 files changed, 110 insertions(+), 7 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index ef1021942..5771a621d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -592,6 +592,21 @@ def generate_simulated_alert_response( original_request = f'Your original request to the agent was: "{question}"\n' if question else "" + # The agent may offer several metrics and describe one as ignoring filters or counting all + # time. That reads like rule 5's "no filters ... all time", which made the simulated user + # pick it. The pick follows the goal's metric whatever its description says, matched by id + # or by title, and repeats the option as the agent wrote it, so the reply adds no reference + # the agent did not offer. A fixture names a bare metric id: `_check_metric` compares it as + # written. + metric_choice_rule = ( + f" If the agent offers several metrics to choose from, choose the option for your goal metric " + f"({expected.metric_id}; the agent may show it by this id or only by its title) and repeat that option " + "as the agent wrote it. How the agent describes a metric, for example as ignoring filters or counting all " + "time, never changes which metric you want: rule 5 is about filters on the alert.\n" + if expected.metric_id + else "" + ) + # Only a stated title is enforced: an unstated one is the agent's to choose, and telling the # sim-user nothing keeps it from objecting to a generated name. title_rule = ( @@ -607,8 +622,11 @@ def generate_simulated_alert_response( + original_request + "Rules you MUST follow:\n" f"1. Your goal: metric={metric}, operator={operator}, threshold={threshold}, " - f"recipients={recipients}, trigger={trigger_key}" + (f", filters={filters}" if filters else "") + ".\n" - "2. Never revert or change a decision that was already confirmed in a previous turn.\n" + f"recipients={recipients}, trigger={trigger_key}" + + (f", filters={filters}" if filters else "") + + ".\n" + + metric_choice_rule + + "2. Never revert or change a decision that was already confirmed in a previous turn.\n" "3. If the agent shows a final summary, an alert proposal or asks for confirmation, check " " ALL of these against your goal: recipients, trigger (how often you are alerted), " " filters / time window, threshold and operator. If ANY of them differs — for example the " diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b6ca2359a..141e64f48 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -108,8 +108,13 @@ class AgenticRunSummary: def generate_simulated_response(agent_message: str, expected_output: CreatedVisualization) -> str: """Generate a simulated user reply to an agent clarification question. - Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent - toward the expected visualization without revealing the answer directly. + Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent toward the + expected visualization without revealing the answer directly. Objects are named in words + made from their ids (`uri_to_display_name`: `metric/total_returns` -> "total returns"), not + by their titles, which are not available here, and never as `{type/id}` references: a goal + of references makes the simulated user volunteer them, handing the agent objects it should + resolve itself. A date filter names its date dataset the same way, without which the + simulated user cannot say which date "2024" means. Requires the [llm-judge] extra: pip install gooddata-eval[llm-judge] """ try: @@ -138,10 +143,12 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu granularity = filter_dict.get("granularity", "") from_val = filter_dict.get("from", "") to_val = filter_dict.get("to", "") + dataset = filter_dict.get("using") + on_dataset = f" on the {uri_to_display_name(dataset)}" if isinstance(dataset, str) and dataset else "" filter_parts.append( - f"date filter: {granularity} from {from_val} to {to_val}" + f"date filter{on_dataset}: {granularity} from {from_val} to {to_val}" if granularity - else f"date filter: {from_val} to {to_val}" + else f"date filter{on_dataset}: {from_val} to {to_val}" ) elif ft == "ranking_filter": n = filter_dict.get("top") or filter_dict.get("bottom") @@ -196,7 +203,9 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu f"Respond naturally to the agent's question. Be helpful and answer what they're asking about.\n" f"If the agent asks specifically about items from your goal (like which metrics or dimensions " f"you want), you should mention them. Keep your response concise and natural, as a real user would." - f"{no_filter_hint}" + f"{no_filter_hint}\n" + f"If the agent offers alternatives, choose the one that matches your goal and repeat it as the " + f"agent wrote it." ), }, ], diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 9513102e3..1cc9ed31d 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -1950,3 +1950,27 @@ def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): assert detail["exit_reason"] == exit_value assert detail["alert_created"] is False + + +# The agent offers metrics as {type/id} and may describe one as ignoring filters, which read +# like rule 5's "no filters"; the simulated user named that metric instead of the fixture's. +def test_sim_user_is_told_which_offered_metric_to_choose() -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": "of_orders"}) + assert "choose the option for your goal metric (of_orders;" in prompt + assert "repeat that option as the agent wrote it" in prompt + assert "never changes which metric you want: rule 5 is about filters on the alert" in prompt + # The reply repeats the agent's option; the prompt hands it no reference to volunteer. + assert "metric/of_orders" not in prompt + + +# When the fixture's metric is the all-time one, nothing in the rule may steer away from it. +def test_sim_user_choice_rule_does_not_argue_against_an_all_time_goal() -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": "orders_all_time"}) + assert "choose the option for your goal metric (orders_all_time;" in prompt + assert "different metric" not in prompt + + +def test_sim_user_without_a_metric_gets_no_metric_choice_rule() -> None: + assert "choose the option for your goal metric" not in _sim_user_prompt( + {"Operator": "LESS_THAN", "Threshold": 1000} + ) diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index c50d2fddb..53108b52f 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -854,3 +854,55 @@ def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkey result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) assert result.exit_reason is loop_exit + + +def _sim_user_call(agent_message: str, viz: CreatedVisualization) -> Any: + """Run the visualization sim-user against a stub OpenAI client and return the call it made.""" + fake_openai = MagicMock() + fake_openai.return_value.chat.completions.create.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content="ok"))] + ) + with patch("openai.OpenAI", fake_openai), patch.dict("os.environ", {"OPENAI_API_KEY": "test-key"}): + agentic_visualization.generate_simulated_response(agent_message, viz) + return fake_openai.return_value.chat.completions.create.call_args + + +def _returns_by_category_in_2024() -> CreatedVisualization: + return CreatedVisualization.model_validate( + { + "id": "v1", + "type": "table", + "query": { + "fields": {"m": {"using": "metric/total_returns"}, "d": {"using": "label/product_category"}}, + "filter_by": { + "f": { + "type": "date_filter", + "using": "dataset/return_date", + "granularity": "YEAR", + "from": "2024-01-01", + "to": "2024-12-31", + } + }, + }, + "metrics": ["m"], + "view_by": ["d"], + } + ) + + +# A date filter without its dataset cannot say whether "2024" is the order or the return date; +# objects given by reference made the simulated user volunteer them to the agent. +def test_sim_user_goal_names_the_date_dataset_and_no_reference() -> None: + prompt = _sim_user_call("Which date should define 2024?", _returns_by_category_in_2024()).kwargs["messages"][1][ + "content" + ] + goal = prompt.split("Your goal")[1] + assert "Metrics: total returns\n" in goal + assert "date filter on the return date: YEAR from 2024-01-01 to 2024-12-31" in goal + assert "{" not in goal + assert "choose the one that matches your goal" in goal + + +def test_sim_user_goal_keeps_dimensions_by_name() -> None: + prompt = _sim_user_call("Which chart?", _returns_by_category_in_2024()).kwargs["messages"][1]["content"] + assert "Dimensions: product category\n" in prompt