From 9cba755cd41e6c44373eeeeb33459f978140fe9f Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Wed, 18 Nov 2026 15:51:48 +0700 Subject: [PATCH] fix(gooddata-eval): sim user prompt names the date dataset and pick When the agent asked which of several objects to use, the visualization and alert simulated users sometimes named the one the fixture does not expect, and the agent built what it was told: Order date vs Return date, new_customers vs new_customers_monthly, of_orders vs orders_all_time. Visualization: the goal's date filter gave only its period, never its date dataset, so a choice between Order date and Return date had no answer in the prompt. It now says "date filter on the return date". The prompt also tells the simulated user to choose the alternative that matches its goal and repeat it as the agent wrote it, and that a date attribute such as order_date.year belongs to the order date. Objects stay named by title: giving them by reference made the simulated user volunteer references to the agent. Alert: the goal has no filters, and rule 5 says "no filters ... all time", which matched an offered metric described as ignoring filters and counting all orders. The prompt now tells the simulated user to choose exactly the goal's metric and that such a metric is a different one. Replayed the three recorded agent messages 20 times each against the real models: all correct, and no reference volunteered to a chart type, open or period question. Models, scoring and gates are unchanged. jira: QA-29478 risk: low Co-Authored-By: Claude Opus 5.5 --- .../gooddata_eval/core/agentic/alert_skill.py | 19 ++++++- .../core/agentic/visualization.py | 17 ++++-- .../tests/test_agentic_alert_skill.py | 13 +++++ .../tests/test_agentic_visualization.py | 52 +++++++++++++++++++ 4 files changed, 94 insertions(+), 7 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index ef1021942..76581b4f7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -592,6 +592,18 @@ def generate_simulated_alert_response( original_request = f'Your original request to the agent was: "{question}"\n' if question else "" + # The agent may offer several metrics and describe one as ignoring filters or counting all + # time. That reads like rule 5's "no filters ... all time", which made the simulated user + # pick it; the metric to name is the goal's, by its reference. + metric_ref = expected.metric_id if "/" in (expected.metric_id or "") else f"metric/{expected.metric_id}" + metric_choice_rule = ( + f" If the agent offers several metrics to choose from, choose exactly {{{metric_ref}}} " + "and write it that way. A metric described as ignoring filters or counting all time is a different " + "metric: rule 5 is about filters on the alert, never a reason to choose another metric.\n" + if expected.metric_id + else "" + ) + # Only a stated title is enforced: an unstated one is the agent's to choose, and telling the # sim-user nothing keeps it from objecting to a generated name. title_rule = ( @@ -607,8 +619,11 @@ def generate_simulated_alert_response( + original_request + "Rules you MUST follow:\n" f"1. Your goal: metric={metric}, operator={operator}, threshold={threshold}, " - f"recipients={recipients}, trigger={trigger_key}" + (f", filters={filters}" if filters else "") + ".\n" - "2. Never revert or change a decision that was already confirmed in a previous turn.\n" + f"recipients={recipients}, trigger={trigger_key}" + + (f", filters={filters}" if filters else "") + + ".\n" + + metric_choice_rule + + "2. Never revert or change a decision that was already confirmed in a previous turn.\n" "3. If the agent shows a final summary, an alert proposal or asks for confirmation, check " " ALL of these against your goal: recipients, trigger (how often you are alerted), " " filters / time window, threshold and operator. If ANY of them differs — for example the " diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b6ca2359a..9caba5090 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -108,8 +108,11 @@ class AgenticRunSummary: def generate_simulated_response(agent_message: str, expected_output: CreatedVisualization) -> str: """Generate a simulated user reply to an agent clarification question. - Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent - toward the expected visualization without revealing the answer directly. + Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent toward the + expected visualization without revealing the answer directly. Objects are named as a user + knows them, never by reference: a goal of references makes the simulated user volunteer + them, handing the agent objects it should resolve itself. A date filter names its date + dataset, without which the simulated user cannot say which date "2024" means. Requires the [llm-judge] extra: pip install gooddata-eval[llm-judge] """ try: @@ -138,10 +141,12 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu granularity = filter_dict.get("granularity", "") from_val = filter_dict.get("from", "") to_val = filter_dict.get("to", "") + dataset = filter_dict.get("using") + on_dataset = f" on the {uri_to_display_name(dataset)}" if isinstance(dataset, str) and dataset else "" filter_parts.append( - f"date filter: {granularity} from {from_val} to {to_val}" + f"date filter{on_dataset}: {granularity} from {from_val} to {to_val}" if granularity - else f"date filter: {from_val} to {to_val}" + else f"date filter{on_dataset}: {from_val} to {to_val}" ) elif ft == "ranking_filter": n = filter_dict.get("top") or filter_dict.get("bottom") @@ -196,7 +201,9 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu f"Respond naturally to the agent's question. Be helpful and answer what they're asking about.\n" f"If the agent asks specifically about items from your goal (like which metrics or dimensions " f"you want), you should mention them. Keep your response concise and natural, as a real user would." - f"{no_filter_hint}" + f"{no_filter_hint}\n" + f"If the agent offers alternatives, choose the one that matches your goal and repeat it as the " + f"agent wrote it. A date attribute such as order_date.year belongs to the order date." ), }, ], diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 9513102e3..5ad7749fe 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -1950,3 +1950,16 @@ def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): assert detail["exit_reason"] == exit_value assert detail["alert_created"] is False + + +# The agent offers metrics as {type/id} and may describe one as ignoring filters, which read +# like rule 5's "no filters"; the simulated user named that metric instead of the fixture's. +@pytest.mark.parametrize("metric_id", ["of_orders", "metric/of_orders"]) +def test_sim_user_is_told_which_offered_metric_to_choose(metric_id: str) -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": metric_id}) + assert "choose exactly {metric/of_orders}" in prompt + assert "rule 5 is about filters on the alert" in prompt + + +def test_sim_user_without_a_metric_gets_no_metric_choice_rule() -> None: + assert "choose exactly" not in _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000}) diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index c50d2fddb..53108b52f 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -854,3 +854,55 @@ def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkey result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) assert result.exit_reason is loop_exit + + +def _sim_user_call(agent_message: str, viz: CreatedVisualization) -> Any: + """Run the visualization sim-user against a stub OpenAI client and return the call it made.""" + fake_openai = MagicMock() + fake_openai.return_value.chat.completions.create.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content="ok"))] + ) + with patch("openai.OpenAI", fake_openai), patch.dict("os.environ", {"OPENAI_API_KEY": "test-key"}): + agentic_visualization.generate_simulated_response(agent_message, viz) + return fake_openai.return_value.chat.completions.create.call_args + + +def _returns_by_category_in_2024() -> CreatedVisualization: + return CreatedVisualization.model_validate( + { + "id": "v1", + "type": "table", + "query": { + "fields": {"m": {"using": "metric/total_returns"}, "d": {"using": "label/product_category"}}, + "filter_by": { + "f": { + "type": "date_filter", + "using": "dataset/return_date", + "granularity": "YEAR", + "from": "2024-01-01", + "to": "2024-12-31", + } + }, + }, + "metrics": ["m"], + "view_by": ["d"], + } + ) + + +# A date filter without its dataset cannot say whether "2024" is the order or the return date; +# objects given by reference made the simulated user volunteer them to the agent. +def test_sim_user_goal_names_the_date_dataset_and_no_reference() -> None: + prompt = _sim_user_call("Which date should define 2024?", _returns_by_category_in_2024()).kwargs["messages"][1][ + "content" + ] + goal = prompt.split("Your goal")[1] + assert "Metrics: total returns\n" in goal + assert "date filter on the return date: YEAR from 2024-01-01 to 2024-12-31" in goal + assert "{" not in goal + assert "choose the one that matches your goal" in goal + + +def test_sim_user_goal_keeps_dimensions_by_name() -> None: + prompt = _sim_user_call("Which chart?", _returns_by_category_in_2024()).kwargs["messages"][1]["content"] + assert "Dimensions: product category\n" in prompt