diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index ef1021942..5771a621d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -592,6 +592,21 @@ def generate_simulated_alert_response( original_request = f'Your original request to the agent was: "{question}"\n' if question else "" + # The agent may offer several metrics and describe one as ignoring filters or counting all + # time. That reads like rule 5's "no filters ... all time", which made the simulated user + # pick it. The pick follows the goal's metric whatever its description says, matched by id + # or by title, and repeats the option as the agent wrote it, so the reply adds no reference + # the agent did not offer. A fixture names a bare metric id: `_check_metric` compares it as + # written. + metric_choice_rule = ( + f" If the agent offers several metrics to choose from, choose the option for your goal metric " + f"({expected.metric_id}; the agent may show it by this id or only by its title) and repeat that option " + "as the agent wrote it. How the agent describes a metric, for example as ignoring filters or counting all " + "time, never changes which metric you want: rule 5 is about filters on the alert.\n" + if expected.metric_id + else "" + ) + # Only a stated title is enforced: an unstated one is the agent's to choose, and telling the # sim-user nothing keeps it from objecting to a generated name. title_rule = ( @@ -607,8 +622,11 @@ def generate_simulated_alert_response( + original_request + "Rules you MUST follow:\n" f"1. Your goal: metric={metric}, operator={operator}, threshold={threshold}, " - f"recipients={recipients}, trigger={trigger_key}" + (f", filters={filters}" if filters else "") + ".\n" - "2. Never revert or change a decision that was already confirmed in a previous turn.\n" + f"recipients={recipients}, trigger={trigger_key}" + + (f", filters={filters}" if filters else "") + + ".\n" + + metric_choice_rule + + "2. Never revert or change a decision that was already confirmed in a previous turn.\n" "3. If the agent shows a final summary, an alert proposal or asks for confirmation, check " " ALL of these against your goal: recipients, trigger (how often you are alerted), " " filters / time window, threshold and operator. If ANY of them differs — for example the " diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b6ca2359a..141e64f48 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -108,8 +108,13 @@ class AgenticRunSummary: def generate_simulated_response(agent_message: str, expected_output: CreatedVisualization) -> str: """Generate a simulated user reply to an agent clarification question. - Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent - toward the expected visualization without revealing the answer directly. + Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent toward the + expected visualization without revealing the answer directly. Objects are named in words + made from their ids (`uri_to_display_name`: `metric/total_returns` -> "total returns"), not + by their titles, which are not available here, and never as `{type/id}` references: a goal + of references makes the simulated user volunteer them, handing the agent objects it should + resolve itself. A date filter names its date dataset the same way, without which the + simulated user cannot say which date "2024" means. Requires the [llm-judge] extra: pip install gooddata-eval[llm-judge] """ try: @@ -138,10 +143,12 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu granularity = filter_dict.get("granularity", "") from_val = filter_dict.get("from", "") to_val = filter_dict.get("to", "") + dataset = filter_dict.get("using") + on_dataset = f" on the {uri_to_display_name(dataset)}" if isinstance(dataset, str) and dataset else "" filter_parts.append( - f"date filter: {granularity} from {from_val} to {to_val}" + f"date filter{on_dataset}: {granularity} from {from_val} to {to_val}" if granularity - else f"date filter: {from_val} to {to_val}" + else f"date filter{on_dataset}: {from_val} to {to_val}" ) elif ft == "ranking_filter": n = filter_dict.get("top") or filter_dict.get("bottom") @@ -196,7 +203,9 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu f"Respond naturally to the agent's question. Be helpful and answer what they're asking about.\n" f"If the agent asks specifically about items from your goal (like which metrics or dimensions " f"you want), you should mention them. Keep your response concise and natural, as a real user would." - f"{no_filter_hint}" + f"{no_filter_hint}\n" + f"If the agent offers alternatives, choose the one that matches your goal and repeat it as the " + f"agent wrote it." ), }, ], diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 9513102e3..1cc9ed31d 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -1950,3 +1950,27 @@ def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): assert detail["exit_reason"] == exit_value assert detail["alert_created"] is False + + +# The agent offers metrics as {type/id} and may describe one as ignoring filters, which read +# like rule 5's "no filters"; the simulated user named that metric instead of the fixture's. +def test_sim_user_is_told_which_offered_metric_to_choose() -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": "of_orders"}) + assert "choose the option for your goal metric (of_orders;" in prompt + assert "repeat that option as the agent wrote it" in prompt + assert "never changes which metric you want: rule 5 is about filters on the alert" in prompt + # The reply repeats the agent's option; the prompt hands it no reference to volunteer. + assert "metric/of_orders" not in prompt + + +# When the fixture's metric is the all-time one, nothing in the rule may steer away from it. +def test_sim_user_choice_rule_does_not_argue_against_an_all_time_goal() -> None: + prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": "orders_all_time"}) + assert "choose the option for your goal metric (orders_all_time;" in prompt + assert "different metric" not in prompt + + +def test_sim_user_without_a_metric_gets_no_metric_choice_rule() -> None: + assert "choose the option for your goal metric" not in _sim_user_prompt( + {"Operator": "LESS_THAN", "Threshold": 1000} + ) diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index c50d2fddb..53108b52f 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -854,3 +854,55 @@ def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkey result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) assert result.exit_reason is loop_exit + + +def _sim_user_call(agent_message: str, viz: CreatedVisualization) -> Any: + """Run the visualization sim-user against a stub OpenAI client and return the call it made.""" + fake_openai = MagicMock() + fake_openai.return_value.chat.completions.create.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content="ok"))] + ) + with patch("openai.OpenAI", fake_openai), patch.dict("os.environ", {"OPENAI_API_KEY": "test-key"}): + agentic_visualization.generate_simulated_response(agent_message, viz) + return fake_openai.return_value.chat.completions.create.call_args + + +def _returns_by_category_in_2024() -> CreatedVisualization: + return CreatedVisualization.model_validate( + { + "id": "v1", + "type": "table", + "query": { + "fields": {"m": {"using": "metric/total_returns"}, "d": {"using": "label/product_category"}}, + "filter_by": { + "f": { + "type": "date_filter", + "using": "dataset/return_date", + "granularity": "YEAR", + "from": "2024-01-01", + "to": "2024-12-31", + } + }, + }, + "metrics": ["m"], + "view_by": ["d"], + } + ) + + +# A date filter without its dataset cannot say whether "2024" is the order or the return date; +# objects given by reference made the simulated user volunteer them to the agent. +def test_sim_user_goal_names_the_date_dataset_and_no_reference() -> None: + prompt = _sim_user_call("Which date should define 2024?", _returns_by_category_in_2024()).kwargs["messages"][1][ + "content" + ] + goal = prompt.split("Your goal")[1] + assert "Metrics: total returns\n" in goal + assert "date filter on the return date: YEAR from 2024-01-01 to 2024-12-31" in goal + assert "{" not in goal + assert "choose the one that matches your goal" in goal + + +def test_sim_user_goal_keeps_dimensions_by_name() -> None: + prompt = _sim_user_call("Which chart?", _returns_by_category_in_2024()).kwargs["messages"][1]["content"] + assert "Dimensions: product category\n" in prompt