Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,21 @@ def generate_simulated_alert_response(

original_request = f'Your original request to the agent was: "{question}"\n' if question else ""

# The agent may offer several metrics and describe one as ignoring filters or counting all
# time. That reads like rule 5's "no filters ... all time", which made the simulated user
# pick it. The pick follows the goal's metric whatever its description says, matched by id
# or by title, and repeats the option as the agent wrote it, so the reply adds no reference
# the agent did not offer. A fixture names a bare metric id: `_check_metric` compares it as
# written.
metric_choice_rule = (
f" If the agent offers several metrics to choose from, choose the option for your goal metric "
f"({expected.metric_id}; the agent may show it by this id or only by its title) and repeat that option "
"as the agent wrote it. How the agent describes a metric, for example as ignoring filters or counting all "
"time, never changes which metric you want: rule 5 is about filters on the alert.\n"
if expected.metric_id
else ""
)

# Only a stated title is enforced: an unstated one is the agent's to choose, and telling the
# sim-user nothing keeps it from objecting to a generated name.
title_rule = (
Expand All @@ -607,8 +622,11 @@ def generate_simulated_alert_response(
+ original_request
+ "Rules you MUST follow:\n"
f"1. Your goal: metric={metric}, operator={operator}, threshold={threshold}, "
f"recipients={recipients}, trigger={trigger_key}" + (f", filters={filters}" if filters else "") + ".\n"
"2. Never revert or change a decision that was already confirmed in a previous turn.\n"
f"recipients={recipients}, trigger={trigger_key}"
+ (f", filters={filters}" if filters else "")
+ ".\n"
+ metric_choice_rule
+ "2. Never revert or change a decision that was already confirmed in a previous turn.\n"
"3. If the agent shows a final summary, an alert proposal or asks for confirmation, check "
" ALL of these against your goal: recipients, trigger (how often you are alerted), "
" filters / time window, threshold and operator. If ANY of them differs — for example the "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,13 @@ class AgenticRunSummary:
def generate_simulated_response(agent_message: str, expected_output: CreatedVisualization) -> str:
"""Generate a simulated user reply to an agent clarification question.

Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent
toward the expected visualization without revealing the answer directly.
Uses OpenAI gpt-5.2 to produce a realistic reply that guides the agent toward the
expected visualization without revealing the answer directly. Objects are named in words
made from their ids (`uri_to_display_name`: `metric/total_returns` -> "total returns"), not
by their titles, which are not available here, and never as `{type/id}` references: a goal
of references makes the simulated user volunteer them, handing the agent objects it should
resolve itself. A date filter names its date dataset the same way, without which the
simulated user cannot say which date "2024" means.
Requires the [llm-judge] extra: pip install gooddata-eval[llm-judge]
"""
try:
Expand Down Expand Up @@ -138,10 +143,12 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu
granularity = filter_dict.get("granularity", "")
from_val = filter_dict.get("from", "")
to_val = filter_dict.get("to", "")
dataset = filter_dict.get("using")
on_dataset = f" on the {uri_to_display_name(dataset)}" if isinstance(dataset, str) and dataset else ""

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uri_to_display_name turns the id into words, but the docstring says objects are named "as a user knows them". For dataset/dt_timeline the goal becomes "date filter on the dt timeline: ...", while the agent asks "Timeline or Closed date?" using the dataset titles. Whenever the id is not the title in snake_case, the sim user has nothing that matches the agent's options, so the ambiguity this PR targets remains.

Is the dataset title available here (e.g. from the catalog or the expected output)?

filter_parts.append(
f"date filter: {granularity} from {from_val} to {to_val}"
f"date filter{on_dataset}: {granularity} from {from_val} to {to_val}"
if granularity
else f"date filter: {from_val} to {to_val}"
else f"date filter{on_dataset}: {from_val} to {to_val}"
)
elif ft == "ranking_filter":
n = filter_dict.get("top") or filter_dict.get("bottom")
Expand Down Expand Up @@ -196,7 +203,9 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu
f"Respond naturally to the agent's question. Be helpful and answer what they're asking about.\n"
f"If the agent asks specifically about items from your goal (like which metrics or dimensions "
f"you want), you should mention them. Keep your response concise and natural, as a real user would."
f"{no_filter_hint}"
f"{no_filter_hint}\n"
f"If the agent offers alternatives, choose the one that matches your goal and repeat it as the "
f"agent wrote it."
),
},
],
Expand Down
24 changes: 24 additions & 0 deletions packages/gooddata-eval/tests/test_agentic_alert_skill.py
Original file line number Diff line number Diff line change
Expand Up @@ -1950,3 +1950,27 @@ def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value):

assert detail["exit_reason"] == exit_value
assert detail["alert_created"] is False


# The agent offers metrics as {type/id} and may describe one as ignoring filters, which read
# like rule 5's "no filters"; the simulated user named that metric instead of the fixture's.
def test_sim_user_is_told_which_offered_metric_to_choose() -> None:
prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": "of_orders"})
assert "choose the option for your goal metric (of_orders;" in prompt
assert "repeat that option as the agent wrote it" in prompt
assert "never changes which metric you want: rule 5 is about filters on the alert" in prompt
# The reply repeats the agent's option; the prompt hands it no reference to volunteer.
assert "metric/of_orders" not in prompt


# When the fixture's metric is the all-time one, nothing in the rule may steer away from it.
def test_sim_user_choice_rule_does_not_argue_against_an_all_time_goal() -> None:
prompt = _sim_user_prompt({"Operator": "LESS_THAN", "Threshold": 1000, "metric_id": "orders_all_time"})
assert "choose the option for your goal metric (orders_all_time;" in prompt
assert "different metric" not in prompt


def test_sim_user_without_a_metric_gets_no_metric_choice_rule() -> None:
assert "choose the option for your goal metric" not in _sim_user_prompt(
{"Operator": "LESS_THAN", "Threshold": 1000}
)
52 changes: 52 additions & 0 deletions packages/gooddata-eval/tests/test_agentic_visualization.py
Original file line number Diff line number Diff line change
Expand Up @@ -854,3 +854,55 @@ def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkey
result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()])

assert result.exit_reason is loop_exit


def _sim_user_call(agent_message: str, viz: CreatedVisualization) -> Any:
"""Run the visualization sim-user against a stub OpenAI client and return the call it made."""
fake_openai = MagicMock()
fake_openai.return_value.chat.completions.create.return_value = MagicMock(
choices=[MagicMock(message=MagicMock(content="ok"))]
)
with patch("openai.OpenAI", fake_openai), patch.dict("os.environ", {"OPENAI_API_KEY": "test-key"}):
agentic_visualization.generate_simulated_response(agent_message, viz)
return fake_openai.return_value.chat.completions.create.call_args


def _returns_by_category_in_2024() -> CreatedVisualization:
return CreatedVisualization.model_validate(
{
"id": "v1",
"type": "table",
"query": {
"fields": {"m": {"using": "metric/total_returns"}, "d": {"using": "label/product_category"}},
"filter_by": {
"f": {
"type": "date_filter",
"using": "dataset/return_date",
"granularity": "YEAR",
"from": "2024-01-01",
"to": "2024-12-31",
}
},
},
"metrics": ["m"],
"view_by": ["d"],
}
)


# A date filter without its dataset cannot say whether "2024" is the order or the return date;
# objects given by reference made the simulated user volunteer them to the agent.
def test_sim_user_goal_names_the_date_dataset_and_no_reference() -> None:
prompt = _sim_user_call("Which date should define 2024?", _returns_by_category_in_2024()).kwargs["messages"][1][
"content"
]
goal = prompt.split("Your goal")[1]
assert "Metrics: total returns\n" in goal
assert "date filter on the return date: YEAR from 2024-01-01 to 2024-12-31" in goal
assert "{" not in goal
assert "choose the one that matches your goal" in goal


def test_sim_user_goal_keeps_dimensions_by_name() -> None:
prompt = _sim_user_call("Which chart?", _returns_by_category_in_2024()).kwargs["messages"][1]["content"]
assert "Dimensions: product category\n" in prompt
Loading