diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 129f4c5cf..3bbca3563 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -20,6 +20,7 @@ from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport @@ -282,6 +283,20 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None: item_report.runs_ungraded = unscored +def _apply_failed_runs(item_report: ItemReport, source: Any) -> None: + """Copy the per-run failure records off an outcome or a failure, if the kind built any. + + Read the same way from both, like the run counts above: a partially passing item raises, + so an item's failing runs reach the report through the exception at least as often as + through the outcome. A kind that builds none keeps the empty default, which reads as + "not instrumented" rather than "nothing failed" -- ``runs_passed``/``runs`` already say + how many failed. + """ + failed_runs = getattr(source, "failed_runs", None) + if failed_runs: + item_report.failed_runs = list(failed_runs) + + def _apply_timings(item_report: ItemReport, timings: Any) -> None: """Copy an outcome's phase breakdown onto the item report, if the kind recorded one. @@ -381,6 +396,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.best_detail = detail or {} _apply_timings(item_report, getattr(outcome, "timings", None)) _apply_run_counts(item_report, outcome) + _apply_failed_runs(item_report, outcome) except AssertionError as exc: item_report.gate_passed = False if gated else None item_report.runs = k @@ -390,11 +406,32 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.best_detail = getattr(exc, "detail", None) or {} _apply_timings(item_report, getattr(exc, "timings", None)) _apply_run_counts(item_report, exc) + _apply_failed_runs(item_report, exc) # Read off the counts, not off the gate: pass^K fails items where runs did pass, # and reporting those as pass_at_k False would contradict the Langfuse score of # the same name. Kinds that report no count read as 0, i.e. a clean failure. item_report.pass_at_k = item_report.runs_passed > 0 print(f"[agentic] {item.id} FAIL: {exc}", flush=True) + except JudgeResponseError as exc: + # Errored, like the branch below -- there is no verdict for any run, so this is + # not K failures -- but NOT diagnostically empty. This is the one failure mode + # where the per-run records are most worth having: the judge broke, so what the + # agent actually said is still there to read, and the conversation ids are how + # anyone gets to it. Reported with the runs it really drove rather than 0. + item_report.error = f"{type(exc).__name__}: {exc}" + item_report.runs = getattr(exc, "runs_effective", None) or k + item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] + item_report.conversation_id = getattr(exc, "conversation_id", None) + item_report.response_id = getattr(exc, "response_id", None) + item_report.best_detail = getattr(exc, "detail", None) or {} + _apply_timings(item_report, getattr(exc, "timings", None)) + _apply_run_counts(item_report, exc) + _apply_failed_runs(item_report, exc) + # Every run ungraded, by definition of this error -- said explicitly rather than + # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a + # kind that attaches no diagnostics would leave at 0. + item_report.runs_ungraded = item_report.runs + print(f"[agentic] {item.id} UNGRADED: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" item_report.runs = 0 diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py new file mode 100644 index 000000000..a324978d5 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py @@ -0,0 +1,62 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Per-run failure records for the agentic evaluators. + +``core/runner.py`` records one of these for every failing run of a single-shot item. The +agentic kinds cannot reuse it: each of them drives its own K-loop inside the evaluator and +hands ``cli/agentic_runner`` a single aggregate per item, so by the time the runner sees +the result the individual attempts are already collapsed into ``best``. The runs themselves +are not lost -- every kind keeps its ``run_results`` list -- they simply never leave the +evaluator. This turns that list into the same records the single-shot path writes. + +Keys mirror ``core.runner._failed_run_record`` so one consumer can read ``failed_runs`` from +either path without branching on test kind. ``tool_call_count``/``tool_names`` are the +addition: the agentic kinds capture tool calls per run, and an agent that produced a final +answer having made no tool call at all answered from the model's own knowledge rather than +from the workspace -- a distinction no other recorded field exposes. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import Any + + +def build_failed_runs( + run_results: Sequence[Any], + *, + passed: Callable[[Any], bool], + detail: Callable[[Any], dict], +) -> list[dict]: + """One record per run that did not pass, in the order the runs happened. + + ``passed`` and ``detail`` are supplied by the caller because neither is uniform across + kinds: guardrail reads ``run.passed`` while visualization reads + ``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything + read here directly goes through ``getattr`` with a default, so a kind whose run result + lacks a field (no response id, no tool capture) records a null instead of raising. + + A run the judge could not grade is recorded like any other non-passing run, with its + ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the + verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``. + """ + records: list[dict] = [] + for run_index, run in enumerate(run_results, start=1): + if passed(run): + continue + tool_calls = list(getattr(run, "tool_call_events", None) or []) + reasoning_steps = list(getattr(run, "reasoning_steps", None) or []) + records.append( + { + "run_index": run_index, + "passed": False, + "error": getattr(run, "judge_error", None), + "detail": detail(run), + "conversation_id": getattr(run, "conversation_id", None), + "response_id": getattr(run, "response_id", None), + "reasoning_step_count": len(reasoning_steps), + "reasoning_steps": reasoning_steps, + "tool_call_count": len(tool_calls), + "tool_names": [getattr(e, "function_name", None) for e in tool_calls], + } + ) + return records diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 8b07ed58d..994848611 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -12,6 +12,7 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -792,6 +793,28 @@ class AlertSkillAssertionError(AgenticAssertionError): """Raised when an alert-skill evaluation fails.""" +def _run_detail(run: AlertRunResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + for this kind that means the full per-field check breakdown and the arguments the run sent. + """ + ev = run.eval + return { + "alert_created": ev.alert_created, + "operator_correct": ev.operator_correct, + "threshold_correct": ev.threshold_correct, + "trigger_correct": ev.trigger_correct, + "filters_correct": ev.filters_correct, + "metric_correct": ev.metric_correct, + "recipients_correct": ev.recipients_correct, + "attributes_correct": ev.attributes_correct, + "granularity_correct": ev.granularity_correct, + "actual_alert_arguments": run.actual_alert_arguments, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + def evaluate_agentic_alert_skill( host: str, token: str, @@ -897,19 +920,14 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.eval - detail = { - "alert_created": ev.alert_created, - "operator_correct": ev.operator_correct, - "threshold_correct": ev.threshold_correct, - "trigger_correct": ev.trigger_correct, - "filters_correct": ev.filters_correct, - "metric_correct": ev.metric_correct, - "recipients_correct": ev.recipients_correct, - "attributes_correct": ev.attributes_correct, - "granularity_correct": ev.granularity_correct, - "actual_alert_arguments": best.actual_alert_arguments, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + detail = _run_detail(best) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.eval.strict_pass, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) @@ -929,6 +947,7 @@ def _write_scores(ctx: RunTraceContext) -> None: exc.detail = detail exc.runs_passed = runs_passed exc.runs_effective = runs_effective + exc.failed_runs = failed_runs raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -937,4 +956,7 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 8c8fb3727..d3e113073 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -6,6 +6,7 @@ import time from dataclasses import dataclass, field +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -214,6 +215,44 @@ class GeneralQuestionAssertionError(AgenticAssertionError): """Raised when a general-question evaluation fails.""" +def _attach_diagnostics( + error: AgenticAssertionError | JudgeResponseError, + best: GeneralQuestionResult, + detail: dict, + failed_runs: list[dict], + runs_passed: int, + runs_effective: int, +) -> None: + """Hang the report payload on a raised error, whichever error it is. + + Both exits carry the same payload: the gate failure below, and the all-ungraded + JudgeResponseError above. They differ only in what the runner does with it -- the + second also marks the item errored -- so the attributes are set in one place. + """ + error.reasoning_steps = best.reasoning_steps + error.conversation_id = best.conversation_id + error.response_id = best.response_id + error.detail = detail + error.runs_passed = runs_passed + error.runs_effective = runs_effective + error.failed_runs = failed_runs + + +def _run_detail(run: GeneralQuestionResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + the two were worth comparing side by side, which they are not if only one of them + carries the judge's reasoning. + """ + return { + "judge_passed": run.passed, + "judge_reasoning": run.reasoning, + "actual_output": run.actual_output, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + def evaluate_agentic_general_question( host: str, token: str, @@ -308,45 +347,51 @@ def _write_scores(ctx: RunTraceContext) -> None: item_timings = sum_timings([r.timings for r in summary.run_results]) unscored = summary.judge_errors - if not summary.scored_run_results: - # Not one run produced a readable verdict, so this item has no result -- an error, - # not K failures. Raised after the trace link is queued so whatever the agent did - # is still linked, and carrying the timings so the runner can report what the item - # cost before it became unevaluable. - exc = JudgeResponseError( - f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " - + " | ".join(unscored) - ) - exc.timings = item_timings - raise exc - + # Computed BEFORE the all-ungraded raise below, not after. An item whose every run went + # ungraded is the one where these matter most -- the judge broke, and the conversation + # ids are what someone needs to go read what the agent actually said -- but the raise + # used to happen first, so the runner caught a bare error and reported runs=0 with no + # records at all. `summary.best` already falls back to the unscored runs, and + # `runs_passed` is then 0, so nothing here needs a graded run to exist. runs_passed = sum(1 for r in summary.scored_run_results if r.passed) runs_effective = len(summary.run_results) best = summary.best detail = { - "judge_passed": best.passed, - "judge_reasoning": best.reasoning, - "actual_output": best.actual_output, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + **_run_detail(best), # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K computed over fewer runs than --runs asked for is a weaker result and # the report has to say so. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), } + # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as + # non-passing here so it is recorded with its judge_error rather than silently dropped. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.judge_error is None and r.passed, + detail=_run_detail, + ) + + if not summary.scored_run_results: + # Not one run produced a readable verdict, so this item has no result -- an error, + # not K failures. Raised after the trace link is queued so whatever the agent did + # is still linked, and carrying the timings so the runner can report what the item + # cost before it became unevaluable. + error = JudgeResponseError( + f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + + " | ".join(unscored) + ) + _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective) + error.timings = item_timings + raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) exc = GeneralQuestionAssertionError( f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id + _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective) exc.timings = item_timings - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -356,4 +401,7 @@ def _write_scores(ctx: RunTraceContext) -> None: response_id=best.response_id, detail=detail, timings=item_timings, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index 255a57299..9f7dcacc7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -187,6 +188,44 @@ class GuardrailAssertionError(AgenticAssertionError): """Raised when a guardrail evaluation fails.""" +def _attach_diagnostics( + error: AgenticAssertionError | JudgeResponseError, + best: GuardrailResult, + detail: dict, + failed_runs: list[dict], + runs_passed: int, + runs_effective: int, +) -> None: + """Hang the report payload on a raised error, whichever error it is. + + Both exits carry the same payload: the gate failure below, and the all-ungraded + JudgeResponseError above. They differ only in what the runner does with it -- the + second also marks the item errored -- so the attributes are set in one place. + """ + error.reasoning_steps = best.reasoning_steps + error.conversation_id = best.conversation_id + error.response_id = best.response_id + error.detail = detail + error.runs_passed = runs_passed + error.runs_effective = runs_effective + error.failed_runs = failed_runs + + +def _run_detail(run: GuardrailResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + the two were worth comparing side by side, which they are not if only one of them + carries the judge's reasoning. + """ + return { + "judge_passed": run.passed, + "judge_reasoning": run.reasoning, + "actual_output": run.actual_output, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + def evaluate_agentic_guardrail( host: str, token: str, @@ -279,39 +318,46 @@ def _write_scores(ctx: RunTraceContext) -> None: unscored = summary.judge_errors - if not summary.scored_run_results: - # No readable verdict for any run: an error, not K failures. Raised after the - # trace link is queued so whatever the agent did is still linked. - raise JudgeResponseError( - f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " - + " | ".join(unscored) - ) - + # Computed BEFORE the all-ungraded raise below, not after. An item whose every run went + # ungraded is the one where these matter most -- the judge broke, and the conversation + # ids are what someone needs to go read what the agent actually said -- but the raise + # used to happen first, so the runner caught a bare error and reported runs=0 with no + # records at all. `summary.best` already falls back to the unscored runs, and + # `runs_passed` is then 0, so nothing here needs a graded run to exist. runs_passed = sum(1 for r in summary.scored_run_results if r.passed) runs_effective = len(summary.run_results) best = summary.best detail = { - "judge_passed": best.passed, - "judge_reasoning": best.reasoning, - "actual_output": best.actual_output, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + **_run_detail(best), # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K over fewer runs than --runs asked for is a weaker result. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), } + # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as + # non-passing here so it is recorded with its judge_error rather than silently dropped. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.judge_error is None and r.passed, + detail=_run_detail, + ) + + if not summary.scored_run_results: + # No readable verdict for any run: an error, not K failures. Raised after the + # trace link is queued so whatever the agent did is still linked. + error = JudgeResponseError( + f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + + " | ".join(unscored) + ) + _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective) + raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) exc = GuardrailAssertionError( f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective + _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective) raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -320,4 +366,7 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 7d7910f62..97da3f961 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -7,6 +7,7 @@ import os from dataclasses import dataclass, field +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -382,6 +383,25 @@ class KdaSkillAssertionError(AgenticAssertionError): """Raised when a KDA-skill evaluation fails.""" +def _run_detail(run: KdaRunResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + for this kind that means which phase the run reached before it stopped. + """ + ev = run.evaluation + return { + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "disambiguated": ev.disambiguated, + "actual_create_args": run.actual_create_args, + "actual_execute_result": run.actual_execute_result, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + def evaluate_agentic_kda_skill( host: str, token: str, @@ -497,16 +517,14 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.evaluation - detail = { - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "disambiguated": ev.disambiguated, - "actual_create_args": best.actual_create_args, - "actual_execute_result": best.actual_execute_result, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + detail = _run_detail(best) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.evaluation.strict_pass, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) @@ -524,6 +542,7 @@ def _write_scores(ctx: RunTraceContext) -> None: exc.detail = detail exc.runs_passed = runs_passed exc.runs_effective = runs_effective + exc.failed_runs = failed_runs raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -532,4 +551,7 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 8c0e777c7..f9412ac34 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -11,6 +11,7 @@ from gooddata_sdk import GoodDataSdk +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -510,13 +511,29 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] - detail = { - "metric_created": best.metric_created, - "maql_correct": best.maql_correct, - "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], - "actual_maql": best.actual_maql, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + + def _run_detail(run: MetricRunResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + for this kind that means the MAQL the run actually produced, which is the whole diagnosis. + """ + return { + "metric_created": run.metric_created, + "maql_correct": run.maql_correct, + "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], + "actual_maql": run.actual_maql, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + detail = _run_detail(best) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.metric_created and r.maql_correct, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) @@ -534,6 +551,7 @@ def _write_scores(ctx: RunTraceContext) -> None: exc.detail = detail exc.runs_passed = runs_passed exc.runs_effective = runs_effective + exc.failed_runs = failed_runs raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -543,4 +561,7 @@ def _write_scores(ctx: RunTraceContext) -> None: response_id=best.response_id, detail=detail, timings=item_timings, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index ac70deeb7..17d4019f3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -168,6 +169,21 @@ class SearchToolAssertionError(AgenticAssertionError): """Raised when a search-tool evaluation fails.""" +def _run_detail(run: SearchResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + for this kind that means the tools it actually called, which is the whole question when + the run failed because it selected the wrong one. + """ + return { + "tool_selected": run.tool_selected, + "tool_correct": run.tool_correct, + "tool_call_names": run.tool_call_names, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + def evaluate_agentic_search_tool( host: str, token: str, @@ -255,12 +271,14 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_effective = len(summary.run_results) best = summary.best - detail = { - "tool_selected": best.tool_selected, - "tool_correct": best.tool_correct, - "tool_call_names": best.tool_call_names, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + detail = _run_detail(best) + # Same predicate pass@K and runs_passed are taken over, so an item's failed_runs and its + # counts cannot disagree about which runs failed. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.tool_selected, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) @@ -275,6 +293,7 @@ def _write_scores(ctx: RunTraceContext) -> None: exc.detail = detail exc.runs_passed = runs_passed exc.runs_effective = runs_effective + exc.failed_runs = failed_runs raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -283,4 +302,7 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 8a26d586e..033ee808c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -10,6 +10,7 @@ import os from dataclasses import dataclass, field +from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, @@ -303,6 +304,19 @@ class VisualizationAssertionError(AgenticAssertionError): """Raised when a visualization evaluation fails.""" +def _run_detail(run: RunResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + for this kind that means the full per-check breakdown, including the expected/actual + filter pairs, which is what tells a wrong filter apart from a wrong metric. + """ + return { + **evaluation_result_detail(run.eval_result), + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + + def _filter_diff(category: str, ev: EvaluationResult) -> str: """Expected-vs-actual lines for one filter category, or "" when they matched. @@ -435,10 +449,16 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.eval_result - detail = { - **evaluation_result_detail(ev), - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + detail = _run_detail(best) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. Every failing run keeps its own + # expected/actual check breakdown -- for this kind that is the whole diagnosis, and + # until now only the winning run's survived. + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.eval_result.strict_pass, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) @@ -483,6 +503,7 @@ def _write_scores(ctx: RunTraceContext) -> None: exc.detail = detail exc.runs_passed = runs_passed exc.runs_effective = runs_effective + exc.failed_runs = failed_runs raise exc return AgenticEvalOutcome( runs_passed=runs_passed, @@ -491,4 +512,7 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + # Also on the success path: pass@K clears the gate with one passing run, so a + # 1/3 item reports success while two of its runs failed for reasons worth reading. + failed_runs=failed_runs, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py index d7e9ddc5f..701898068 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py @@ -49,6 +49,18 @@ class JudgeResponseError(RuntimeError): # rather than attached loosely, because the runner reads it off the exception to report # what an unevaluable item still cost. timings: PhaseTimings + # The rest of that payload, for the same reason. An item whose every run went ungraded + # is still worth reading -- the judge breaking says nothing about what the agent did -- + # so the agentic evaluators attach their per-run records here and cli/agentic_runner + # reports them. Mirrors AgenticAssertionError, which declares the identical set; the two + # cannot share a base, one being an AssertionError and the other a RuntimeError. + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict + runs_passed: int + runs_effective: int + failed_runs: list[dict] def _message_content(response: Any) -> str | None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index a1f1d5165..381b1472a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -288,6 +288,7 @@ class AgenticAssertionError(AssertionError): timings: PhaseTimings runs_passed: int runs_effective: int + failed_runs: list[dict] class AgenticEvalOutcome(BaseModel): @@ -313,6 +314,12 @@ class AgenticEvalOutcome(BaseModel): # ran -- agentic_conversation drives its fixture once whatever --runs says. runs_passed: int = 0 runs_effective: int = 0 + # One record per run that did not pass. ``detail`` above describes only the winning run, + # so without this a 1/K item and a 0/K item carry identical diagnostics and neither says + # anything about the K-1 attempts that failed. Built by + # ``core.agentic._failed_runs.build_failed_runs``; mirrors the single-shot path's + # ``ItemReport.failed_runs``, which ``core/runner.py`` fills for the non-agentic kinds. + failed_runs: list[dict] = Field(default_factory=list) class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 72b3dc349..55d922864 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -70,6 +70,12 @@ def _build_run_dict(report: EvalReport) -> dict: "conversation_id": item.conversation_id, "response_id": item.response_id, "reasoning": item.reasoning_steps, + # Beside `detail`, never merged into it: `detail` keeps its exact meaning + # (the winning run), so every existing consumer of this report is + # unaffected. This is what a partial pass costs you today -- a 1-of-3 shows + # only the attempt that worked -- and each entry carries the ids of its own + # run, which the top-level pair above cannot. + "failed_runs": item.failed_runs, } for item in report.items }, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 039361d2a..7e62dc499 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -45,6 +45,18 @@ class ItemReport: conversation_id: str | None = None response_id: str | None = None reasoning_steps: list[str] = field(default_factory=list) + # One entry per run that did NOT pass, in run order. best_detail describes the winning + # run, so on a 1-of-3 item every visible verdict belongs to the attempt that worked and + # the two that failed leave no trace at all -- their `detail` is computed here and then + # dropped. That makes a partial pass undiagnosable after the fact: the only recourse is + # re-running the question and hoping it fails the same way. + # + # Failing runs only, deliberately. A fully-passing item adds nothing, so the cost tracks + # how broken the corpus is and shrinks as it improves. Each entry also carries its OWN + # conversation_id/response_id: the report's top-level pair is overwritten every run and + # ends up describing the LAST one, which is not necessarily the run best_detail is + # about, so those ids cannot be used to pull the trace for a specific failure. + failed_runs: list[dict] = field(default_factory=list) # Per-phase breakdown of what the item's time was spent on. Additive to latency_s, # which remains the item's own critical path. langfuse_latency_s is # deliberately NOT part of that path -- trace linking runs off it (see @@ -175,6 +187,32 @@ def avg_quality_score(self) -> float: RunCallback = Callable[[int, int, bool, float], None] +def _failed_run_record(run_index: int, evaluation: ItemEvaluation, chat_result: ChatResult, latency: float) -> dict: + """Everything needed to diagnose ONE failing run, without re-running it. + + `detail` is the evaluator's own verdict for this attempt, opaque here -- the runner + never inspects its shape, so this works for every test kind and for kinds added later. + + `stream_ended` separates a stalled turn from a wrong answer. A stall leaves the gated + checks False even though none of them ran, which reads as a content failure in every + downstream rate; this records the difference at the source instead of leaving consumers + to infer it. + """ + return { + "run_index": run_index, + "passed": False, + "error": evaluation.error, + "detail": evaluation.detail, + "conversation_id": getattr(chat_result, "conversation_id", None), + "response_id": getattr(chat_result, "response_id", None), + "stream_ended": getattr(chat_result, "stream_ended", None), + "turn_wall_clock_sec": getattr(chat_result, "turn_wall_clock_sec", None), + "latency_s": round(latency, 3), + "reasoning_step_count": getattr(chat_result, "reasoning_step_count", 0), + "reasoning_steps": list(getattr(chat_result, "reasoning_steps", None) or []), + } + + def _run_one_item( item: DatasetItem, backend: ChatBackend, runs: int, on_run_done: RunCallback | None = None ) -> ItemReport: @@ -212,6 +250,8 @@ def _run_one_item( if evaluation.passed: report.pass_at_k = True report.runs_passed += 1 + else: + report.failed_runs.append(_failed_run_record(run_index, evaluation, chat_result, latency)) if on_run_done is not None: on_run_done(run_index, runs, evaluation.passed, latency) except Exception as e: # agent/network/parse failure for this item diff --git a/packages/gooddata-eval/tests/test_agentic_general_question.py b/packages/gooddata-eval/tests/test_agentic_general_question.py index 06609f21d..8a2460fdd 100644 --- a/packages/gooddata-eval/tests/test_agentic_general_question.py +++ b/packages/gooddata-eval/tests/test_agentic_general_question.py @@ -710,6 +710,14 @@ def test_an_item_with_no_gradeable_run_raises_instead_of_reporting_failures(): # Carried so the runner can still report what the item cost before it became # unevaluable. assert err.value.timings.agent_s == 7.0 # 4.0 + 3.0 + # And the per-run records, for the same reason. The judge breaking says nothing about + # what the agent did, so an item with no verdict at all is still worth reading -- these + # carry each run's own output and conversation id. Asserted behaviourally because the + # cross-kind source check cannot see whether this branch attaches them: its + # `_attach_diagnostics(` match is also satisfied by the helper's own definition. + assert [r["run_index"] for r in err.value.failed_runs] == [1, 2] + assert [r["error"] for r in err.value.failed_runs] == ["empty body twice", "no 'score' key"] + assert (err.value.runs_passed, err.value.runs_effective) == (0, 2) def test_run_agentic_general_question_forwards_the_user_context_to_the_chat_client(): diff --git a/packages/gooddata-eval/tests/test_agentic_guardrail.py b/packages/gooddata-eval/tests/test_agentic_guardrail.py index 320ba7b9a..80357239e 100644 --- a/packages/gooddata-eval/tests/test_agentic_guardrail.py +++ b/packages/gooddata-eval/tests/test_agentic_guardrail.py @@ -11,6 +11,7 @@ evaluate_agentic_guardrail, run_agentic_guardrail, ) +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult @@ -234,3 +235,120 @@ def test_a_non_unanimous_pass_reaches_the_outcome(): ) assert (outcome.runs_passed, outcome.runs_effective) == (2, 3) + + +# --- every failing run's detail has to survive, not just the winning one's --- + + +def _client_and_judge_with_tools(verdicts, tool_names_per_run): + """Like ``_guardrail_client_and_judge`` but each run reports its own tool calls. + + Tool calls are what separate an agent that consulted the workspace from one that + answered out of the model's own knowledge, so ``failed_runs`` has to carry them per run. + """ + client = MagicMock() + client.create_conversation.side_effect = (f"conv-{i}" for i in itertools.count(1)) + tools = iter(tool_names_per_run) + client.send_message.side_effect = lambda c, q, **k: ChatResult.model_validate( + { + "textResponse": f"answer {c}", + "toolCallEvents": [{"functionName": name, "functionArguments": "{}"} for name in next(tools)], + "reasoningSteps": [], + "responseId": "r", + } + ) + it = iter(verdicts) + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.side_effect = lambda **kw: next(it) + return client, judge + + +def test_failed_runs_records_only_the_runs_that_did_not_pass(): + """A 1/3 item and a 0/3 item used to carry identical diagnostics: the winner's detail.""" + client, judge = _guardrail_client_and_judge([(True, "ok"), (False, "answered it"), (False, "answered again")]) + + with _patched(client, judge): + outcome = evaluate_agentic_guardrail( + host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=3 + ) + + assert [r["run_index"] for r in outcome.failed_runs] == [2, 3] + assert all(r["passed"] is False for r in outcome.failed_runs) + # Each failing run's OWN verdict, not the best run's -- the point of keeping them. + assert [r["detail"]["judge_reasoning"] for r in outcome.failed_runs] == ["answered it", "answered again"] + assert [r["detail"]["actual_output"] for r in outcome.failed_runs] == ["answer conv-2", "answer conv-3"] + # Distinct conversation ids: the latent pairing bug was reporting one id for the item. + assert [r["conversation_id"] for r in outcome.failed_runs] == ["conv-2", "conv-3"] + + +def test_failed_runs_is_empty_when_every_run_passed(): + client, judge = _guardrail_client_and_judge([(True, "ok")] * 3) + + with _patched(client, judge): + outcome = evaluate_agentic_guardrail( + host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=3 + ) + + assert outcome.failed_runs == [] + + +def test_failed_runs_reaches_the_exception_when_no_run_passed(): + client, judge = _guardrail_client_and_judge([(False, "answered it")] * 2) + + with _patched(client, judge), pytest.raises(GuardrailAssertionError) as exc_info: + evaluate_agentic_guardrail(host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=2) + + assert [r["run_index"] for r in exc_info.value.failed_runs] == [1, 2] + + +def test_failed_runs_records_an_ungraded_run_with_its_judge_error(): + """An ungraded run has no verdict, so it is not a pass -- and dropping it would hide + the one run whose failure was the judge's, not the agent's. + """ + client, judge = _guardrail_client_and_judge([(True, "ok")] * 2) + judge.score.side_effect = [(True, "ok"), JudgeResponseError("unparseable")] + + with _patched(client, judge): + outcome = evaluate_agentic_guardrail( + host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=2 + ) + + assert [(r["run_index"], r["error"]) for r in outcome.failed_runs] == [(2, "unparseable")] + + +def test_failed_runs_carries_each_run_s_tool_calls(): + """No tool call at all means the answer came from the model, not the workspace.""" + client, judge = _client_and_judge_with_tools( + [(True, "ok"), (False, "answered it")], + [["search_catalog"], []], + ) + + with _patched(client, judge): + outcome = evaluate_agentic_guardrail( + host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=2 + ) + + assert [(r["tool_call_count"], r["tool_names"]) for r in outcome.failed_runs] == [(0, [])] + + +# --- an item whose every run went ungraded is still diagnosable --- + + +def test_all_ungraded_attaches_the_records_to_the_judge_error(): + """The judge broke, so there is no verdict -- but what the agent said is still there, + and the conversation ids are how anyone gets to it. This used to raise before the + records were built, so the runner reported the item with nothing at all. + """ + client, judge = _guardrail_client_and_judge([]) + judge.score.side_effect = [JudgeResponseError("unparseable"), JudgeResponseError("unparseable")] + + with pytest.raises(JudgeResponseError) as exc_info, _patched(client, judge): + evaluate_agentic_guardrail(host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=2) + + err = exc_info.value + assert [r["run_index"] for r in err.failed_runs] == [1, 2] + assert [r["error"] for r in err.failed_runs] == ["unparseable", "unparseable"] + assert [r["conversation_id"] for r in err.failed_runs] == ["conv-1", "conv-2"] + assert [r["detail"]["actual_output"] for r in err.failed_runs] == ["answer conv-1", "answer conv-2"] + assert (err.runs_passed, err.runs_effective) == (0, 2) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 77086d101..3f18633b9 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,11 +1,14 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import importlib +import inspect import threading import time from concurrent.futures import ThreadPoolExecutor from unittest.mock import patch import pytest +from gooddata_eval.cli import agentic_runner from gooddata_eval.cli.agentic_runner import ( AGENTIC_TEST_KINDS, PARALLEL_SAFE_TEST_KINDS, @@ -15,6 +18,7 @@ runs_in_parallel, ) from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem from gooddata_eval.core.timing import PhaseTimings @@ -766,3 +770,115 @@ def test_dispatch_agentic_passes_user_context_through_to_general_question(): model_version_override=None, ) assert mock_eval.call_args.kwargs["user_context"] == attachment + + +# --- failed_runs has to reach the report, from the outcome AND from the failure --- + +_A_FAILED_RUN = { + "run_index": 2, + "passed": False, + "error": None, + "detail": {"judge_reasoning": "answered it"}, + "conversation_id": "conv-2", + "response_id": "resp-2", + "reasoning_step_count": 0, + "reasoning_steps": [], + "tool_call_count": 0, + "tool_names": [], +} + + +def test_run_agentic_items_surfaces_failed_runs_on_a_partial_pass(): + """pass@K clears the gate on one passing run, so this item succeeds -- and its failing + runs reach the report only through this field.""" + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=AgenticEvalOutcome( + reasoning_steps=[], detail={"alert_created": True}, runs_passed=1, failed_runs=[_A_FAILED_RUN] + ), + ): + report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") + assert report.items[0].pass_at_k is True + assert report.items[0].failed_runs == [_A_FAILED_RUN] + + +def test_run_agentic_items_surfaces_failed_runs_from_the_exception_on_fail(): + exc = AlertSkillAssertionError("nope") + exc.reasoning_steps = [] + exc.detail = {"alert_created": False} + exc.failed_runs = [_A_FAILED_RUN] + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") + assert report.items[0].failed_runs == [_A_FAILED_RUN] + + +def test_a_kind_that_records_no_failed_runs_keeps_the_empty_default(): + """Absent, not invented: an empty list reads as "not instrumented", and runs_passed vs + runs already says how many runs failed.""" + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=AgenticEvalOutcome(reasoning_steps=[], detail={}), + ): + report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") + assert report.items[0].failed_runs == [] + + +@pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) +def test_failed_runs_reaches_the_report_for_every_kind(kind, expected_output, target): + """The regression test for the gap this closes: `failed_runs` shipped on ItemReport and + in the JSON, but only `core/runner.py` ever filled it, so every agentic kind wrote the + field empty on every item (confirmed live: 135 agentic_guardrail results, all with + `failed_runs: []`, including 16 items that passed 1 of 3 runs). A kind whose evaluator + stops attaching them fails here rather than quietly reporting nothing.""" + item = DatasetItem(id="q1", dataset_name="ds", test_kind=kind, question="q", expected_output=expected_output) + canned = AgenticEvalOutcome(reasoning_steps=["x"], detail={"k": "v"}, failed_runs=[_A_FAILED_RUN]) + with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned): + report = run_agentic_items([item], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") + assert report.items[0].failed_runs == [_A_FAILED_RUN] + + +# Kinds that legitimately build no per-run failure records. agentic_conversation drives its +# fixture exactly once whatever --runs says (it is the sole member of +# UNGATED_AGENTIC_TEST_KINDS), so it has no K to have failing runs within. +_KINDS_WITHOUT_FAILED_RUNS = {"agentic_conversation"} + + +@pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) +def test_every_multi_run_kind_s_evaluator_builds_failed_runs(kind, expected_output, target): + """Structural, because the test above cans the outcome and so cannot see whether the + evaluator filled it. Each K-running evaluator has to actually call build_failed_runs; + without this, a new kind reaches production writing `failed_runs: []` on every item and + nothing fails -- which is exactly how the gap this closes survived a full release.""" + module = importlib.import_module(getattr(agentic_runner, target).__module__) + source = inspect.getsource(module) + if kind in _KINDS_WITHOUT_FAILED_RUNS: + pytest.skip(f"{kind} runs its fixture once; no K to fail within") + assert "build_failed_runs(" in source, f"{module.__name__} never builds per-run failure records" + assert "failed_runs=failed_runs" in source, f"{module.__name__} never returns them on the success path" + # Either set directly on the raised error, or via the kind's own _attach_diagnostics + # helper -- the judge-based kinds raise from two places and set the payload in one. + attaches = "failed_runs = failed_runs" in source or "_attach_diagnostics(" in source + assert attaches, f"{module.__name__} never attaches them to its failure" + + +def test_an_all_ungraded_item_is_errored_but_still_carries_its_failed_runs(): + """JudgeResponseError is a RuntimeError, so it used to land in the generic branch: + errored, runs=0, no records. It is still an error -- no run has a verdict -- but the + per-run diagnostics are precisely what makes a broken judge investigable.""" + err = JudgeResponseError("judge returned no readable verdict for any of the 3 run(s)") + err.failed_runs = [_A_FAILED_RUN] + err.conversation_id = "conv-2" + err.detail = {"actual_output": "something the agent said"} + err.runs_passed = 0 + err.runs_effective = 3 + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=err): + report = run_agentic_items( + [_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01", k=3 + ) + item = report.items[0] + assert item.error is not None + assert item.failed_runs == [_A_FAILED_RUN] + assert item.conversation_id == "conv-2" + assert item.best_detail == {"actual_output": "something the agent said"} + # The runs it really drove, not 0, and every one of them ungraded. + assert (item.runs, item.runs_ungraded) == (3, 3) diff --git a/packages/gooddata-eval/tests/test_reporting.py b/packages/gooddata-eval/tests/test_reporting.py index 7f582618b..8662b62c9 100644 --- a/packages/gooddata-eval/tests/test_reporting.py +++ b/packages/gooddata-eval/tests/test_reporting.py @@ -442,3 +442,49 @@ def test_a_failed_item_says_when_a_criterion_went_ungraded(): out = _rendered(report) assert "did not pass strict checks; 1 criterion(s) ungraded" in out + + +def test_json_report_carries_failed_runs_beside_the_winning_detail(): + """`detail` keeps its exact meaning -- the winning run -- so every existing consumer of + this report is unaffected; the failing attempts arrive alongside it rather than + replacing it.""" + report = EvalReport(model="gpt-5.2") + report.items.append( + ItemReport( + id="i1", + dataset_name="d", + test_kind="agentic_dashboard_summary", + question="q", + pass_at_k=True, + runs=2, + runs_passed=1, + best_detail={"rubric_0": True}, + conversation_id="conv-2", + failed_runs=[ + { + "run_index": 1, + "passed": False, + "error": None, + "detail": {"rubric_0": False}, + "conversation_id": "conv-1", + "response_id": "resp-1", + "stream_ended": True, + "reasoning_steps": ["why it went wrong"], + } + ], + ) + ) + + item = build_json_report(report)["items"]["i1"] + assert item["detail"] == {"rubric_0": True} + assert item["conversation_id"] == "conv-2" + assert [r["detail"] for r in item["failed_runs"]] == [{"rubric_0": False}] + assert item["failed_runs"][0]["conversation_id"] == "conv-1" + + +def test_json_report_failed_runs_is_empty_for_a_clean_item(): + report = EvalReport(model="gpt-5.2") + report.items.append( + ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=2) + ) + assert build_json_report(report)["items"]["i1"]["failed_runs"] == [] diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index 64c5c3567..c661b0672 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -427,3 +427,86 @@ def test_best_detail_describes_a_graded_run_when_there_is_one(): report, _ = _run_scripted([_ungraded(), _graded(False)], runs=2) assert report.items[0].best_detail == {"judge_passed": False} + + +def _chat_with_ids(conversation_id: str, response_id: str, *, stream_ended: bool = True) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": "which metric?", + "conversationId": conversation_id, + "responseId": response_id, + "streamEnded": stream_ended, + "reasoningSteps": [f"thinking in {conversation_id}"], + "reasoningStepCount": 1, + } + ) + + +def test_a_partial_pass_keeps_the_detail_of_the_run_that_failed(): + """The gap this closes: best_detail describes the winning run, so a 1-of-2 item used to + expose only the attempt that worked and the failure left no trace to diagnose.""" + report, _ = _run_scripted([_graded(False), _graded(True)], runs=2) + + item = report.items[0] + assert item.pass_at_k is True and item.runs_passed == 1 + assert item.best_detail == {"judge_passed": True}, "unchanged: still the winning run" + assert [r["detail"] for r in item.failed_runs] == [{"judge_passed": False}] + assert item.failed_runs[0]["run_index"] == 1 + + +def test_a_fully_passing_item_records_no_failed_runs(): + """Failing runs only -- the cost tracks how broken the corpus is, not how large it is.""" + report, _ = _run_scripted([_graded(True), _graded(True)], runs=2) + + assert report.items[0].pass_power_k is True + assert report.items[0].failed_runs == [] + + +def test_every_failing_run_is_recorded_in_run_order(): + report, _ = _run_scripted([_graded(False), _graded(True), _graded(False)], runs=3) + + assert [r["run_index"] for r in report.items[0].failed_runs] == [1, 3] + + +def test_a_failed_run_carries_the_ids_of_its_own_conversation(): + """The report's top-level pair is overwritten every run and ends up describing the LAST + one, which need not be the run best_detail is about. Pulling the trace for a specific + failure needs that failure's own ids.""" + backend = _FakeBackend([_chat_with_ids("conv-1", "resp-1"), _chat_with_ids("conv-2", "resp-2")]) + with patch( + "gooddata_eval.core.runner.get_evaluator", + return_value=_ScriptedEvaluator([_graded(False), _graded(True)]), + ): + report = run_items([_item()], backend, runs=2) + + item = report.items[0] + assert item.conversation_id == "conv-2", "top-level still describes the last run" + assert item.failed_runs[0]["conversation_id"] == "conv-1" + assert item.failed_runs[0]["response_id"] == "resp-1" + assert item.failed_runs[0]["reasoning_steps"] == ["thinking in conv-1"] + + +def test_a_failed_run_records_whether_the_turn_actually_finished(): + """A stall leaves the evaluator's gated checks False even though none of them ran, which + reads as a content failure downstream. stream_ended records the difference at source.""" + backend = _FakeBackend([_chat_with_ids("conv-1", "resp-1", stream_ended=False)]) + with patch( + "gooddata_eval.core.runner.get_evaluator", + return_value=_ScriptedEvaluator([_graded(False)]), + ): + report = run_items([_item()], backend, runs=1) + + failed = report.items[0].failed_runs[0] + assert failed["stream_ended"] is False + assert failed["reasoning_step_count"] == 1 + assert failed["latency_s"] >= 0 + + +def test_an_ungraded_run_is_recorded_as_a_failure_with_its_judge_error(): + """Never a pass, so it belongs here -- and its judge error is the only thing that + explains why pass_power_k is False on an item whose graded runs all passed.""" + report, _ = _run_scripted([_graded(True), _ungraded()], runs=2) + + item = report.items[0] + assert item.pass_at_k is True and item.pass_power_k is False + assert [r["error"] for r in item.failed_runs] == ["empty body"]