Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,18 @@ def generate_simulated_alert_response(

original_request = f'Your original request to the agent was: "{question}"\n' if question else ""

# The agent may offer several metrics and describe one as ignoring filters or counting all
# time. That reads like rule 5's "no filters ... all time", which made the simulated user
# pick it; the metric to name is the goal's, by its reference.
metric_ref = expected.metric_id if "/" in (expected.metric_id or "") else f"metric/{expected.metric_id}"
metric_choice_rule = (
f" If the agent offers several metrics to choose from, choose exactly {{{metric_ref}}} "
"and write it that way. A metric described as ignoring filters or counting all time is a different "
"metric: rule 5 is about filters on the alert, never a reason to choose another metric.\n"
if expected.metric_id
else ""
)

# Only a stated title is enforced: an unstated one is the agent's to choose, and telling the
# sim-user nothing keeps it from objecting to a generated name.
title_rule = (
Expand All @@ -607,8 +619,11 @@ def generate_simulated_alert_response(
+ original_request
+ "Rules you MUST follow:\n"
f"1. Your goal: metric={metric}, operator={operator}, threshold={threshold}, "
f"recipients={recipients}, trigger={trigger_key}" + (f", filters={filters}" if filters else "") + ".\n"
"2. Never revert or change a decision that was already confirmed in a previous turn.\n"
f"recipients={recipients}, trigger={trigger_key}"
+ (f", filters={filters}" if filters else "")
+ ".\n"
+ metric_choice_rule
+ "2. Never revert or change a decision that was already confirmed in a previous turn.\n"
"3. If the agent shows a final summary, an alert proposal or asks for confirmation, check "
" ALL of these against your goal: recipients, trigger (how often you are alerted), "
" filters / time window, threshold and operator. If ANY of them differs — for example the "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,11 @@ class AgenticRunSummary:
def generate_simulated_response(agent_message: str, expected_output: CreatedVisualization) -> str:
"""Generate a simulated user reply to an agent clarification question.

Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent
toward the expected visualization without revealing the answer directly.
Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent toward the
expected visualization without revealing the answer directly. Objects are named as a user
knows them, never by reference: a goal of references makes the simulated user volunteer
them, handing the agent objects it should resolve itself. A date filter names its date
dataset, without which the simulated user cannot say which date "2024" means.
Requires the [llm-judge] extra: pip install gooddata-eval[llm-judge]
"""
try:
Expand Down Expand Up @@ -138,10 +141,12 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu
granularity = filter_dict.get("granularity", "")
from_val = filter_dict.get("from", "")
to_val = filter_dict.get("to", "")
dataset = filter_dict.get("using")
on_dataset = f" on the {uri_to_display_name(dataset)}" if isinstance(dataset, str) and dataset else ""
filter_parts.append(
f"date filter: {granularity} from {from_val} to {to_val}"
f"date filter{on_dataset}: {granularity} from {from_val} to {to_val}"
if granularity
else f"date filter: {from_val} to {to_val}"
else f"date filter{on_dataset}: {from_val} to {to_val}"
)
elif ft == "ranking_filter":
n = filter_dict.get("top") or filter_dict.get("bottom")
Expand Down Expand Up @@ -196,7 +201,9 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu
f"Respond naturally to the agent's question. Be helpful and answer what they're asking about.\n"
f"If the agent asks specifically about items from your goal (like which metrics or dimensions "
f"you want), you should mention them. Keep your response concise and natural, as a real user would."
f"{no_filter_hint}"
f"{no_filter_hint}\n"
f"If the agent offers alternatives, choose the one that matches your goal and repeat it as the "
f"agent wrote it. A date attribute such as order_date.year belongs to the order date."
),
},
],
Expand Down
13 changes: 13 additions & 0 deletions packages/gooddata-eval/tests/test_agentic_alert_skill.py
Original file line number Diff line number Diff line change
Expand Up @@ -1950,3 +1950,16 @@ def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value):

assert detail["exit_reason"] == exit_value
assert detail["alert_created"] is False


# The agent offers metrics as {type/id} and may describe one as ignoring filters, which read
# like rule 5's "no filters"; the simulated user named that metric instead of the fixture's.
@pytest.mark.parametrize("metric_id", ["of_orders", "metric/of_orders"])
def test_sim_user_is_told_which_offered_metric_to_choose(metric_id: str) -> None:
prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": metric_id})
assert "choose exactly {metric/of_orders}" in prompt
assert "rule 5 is about filters on the alert" in prompt


def test_sim_user_without_a_metric_gets_no_metric_choice_rule() -> None:
assert "choose exactly" not in _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000})
52 changes: 52 additions & 0 deletions packages/gooddata-eval/tests/test_agentic_visualization.py
Original file line number Diff line number Diff line change
Expand Up @@ -854,3 +854,55 @@ def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkey
result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()])

assert result.exit_reason is loop_exit


def _sim_user_call(agent_message: str, viz: CreatedVisualization) -> Any:
"""Run the visualization sim-user against a stub OpenAI client and return the call it made."""
fake_openai = MagicMock()
fake_openai.return_value.chat.completions.create.return_value = MagicMock(
choices=[MagicMock(message=MagicMock(content="ok"))]
)
with patch("openai.OpenAI", fake_openai), patch.dict("os.environ", {"OPENAI_API_KEY": "test-key"}):
agentic_visualization.generate_simulated_response(agent_message, viz)
return fake_openai.return_value.chat.completions.create.call_args


def _returns_by_category_in_2024() -> CreatedVisualization:
return CreatedVisualization.model_validate(
{
"id": "v1",
"type": "table",
"query": {
"fields": {"m": {"using": "metric/total_returns"}, "d": {"using": "label/product_category"}},
"filter_by": {
"f": {
"type": "date_filter",
"using": "dataset/return_date",
"granularity": "YEAR",
"from": "2024-01-01",
"to": "2024-12-31",
}
},
},
"metrics": ["m"],
"view_by": ["d"],
}
)


# A date filter without its dataset cannot say whether "2024" is the order or the return date;
# objects given by reference made the simulated user volunteer them to the agent.
def test_sim_user_goal_names_the_date_dataset_and_no_reference() -> None:
prompt = _sim_user_call("Which date should define 2024?", _returns_by_category_in_2024()).kwargs["messages"][1][
"content"
]
goal = prompt.split("Your goal")[1]
assert "Metrics: total returns\n" in goal
assert "date filter on the return date: YEAR from 2024-01-01 to 2024-12-31" in goal
assert "{" not in goal
assert "choose the one that matches your goal" in goal


def test_sim_user_goal_keeps_dimensions_by_name() -> None:
prompt = _sim_user_call("Which chart?", _returns_by_category_in_2024()).kwargs["messages"][1]["content"]
assert "Dimensions: product category\n" in prompt
Loading