feat(eval): capture the detail of every failing run, not just the winning one

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -20,6 +20,7 @@
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -282,6 +283,20 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None:
         item_report.runs_ungraded = unscored
 
 
+def _apply_failed_runs(item_report: ItemReport, source: Any) -> None:
+    """Copy the per-run failure records off an outcome or a failure, if the kind built any.
+
+    Read the same way from both, like the run counts above: a partially passing item raises,
+    so an item's failing runs reach the report through the exception at least as often as
+    through the outcome. A kind that builds none keeps the empty default, which reads as
+    "not instrumented" rather than "nothing failed" -- ``runs_passed``/``runs`` already say
+    how many failed.
+    """
+    failed_runs = getattr(source, "failed_runs", None)
+    if failed_runs:
+        item_report.failed_runs = list(failed_runs)
+
+
 def _apply_timings(item_report: ItemReport, timings: Any) -> None:
     """Copy an outcome's phase breakdown onto the item report, if the kind recorded one.
 
@@ -381,6 +396,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             item_report.best_detail = detail or {}
             _apply_timings(item_report, getattr(outcome, "timings", None))
             _apply_run_counts(item_report, outcome)
+            _apply_failed_runs(item_report, outcome)
         except AssertionError as exc:
             item_report.gate_passed = False if gated else None
             item_report.runs = k
@@ -390,11 +406,32 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             item_report.best_detail = getattr(exc, "detail", None) or {}
             _apply_timings(item_report, getattr(exc, "timings", None))
             _apply_run_counts(item_report, exc)
+            _apply_failed_runs(item_report, exc)
             # Read off the counts, not off the gate: pass^K fails items where runs did pass,
             # and reporting those as pass_at_k False would contradict the Langfuse score of
             # the same name. Kinds that report no count read as 0, i.e. a clean failure.
             item_report.pass_at_k = item_report.runs_passed > 0
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
+        except JudgeResponseError as exc:
+            # Errored, like the branch below -- there is no verdict for any run, so this is
+            # not K failures -- but NOT diagnostically empty. This is the one failure mode
+            # where the per-run records are most worth having: the judge broke, so what the
+            # agent actually said is still there to read, and the conversation ids are how
+            # anyone gets to it. Reported with the runs it really drove rather than 0.
+            item_report.error = f"{type(exc).__name__}: {exc}"
+            item_report.runs = getattr(exc, "runs_effective", None) or k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
+            item_report.conversation_id = getattr(exc, "conversation_id", None)
+            item_report.response_id = getattr(exc, "response_id", None)
+            item_report.best_detail = getattr(exc, "detail", None) or {}
+            _apply_timings(item_report, getattr(exc, "timings", None))
+            _apply_run_counts(item_report, exc)
+            _apply_failed_runs(item_report, exc)
+            # Every run ungraded, by definition of this error -- said explicitly rather than
+            # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a
+            # kind that attaches no diagnostics would leave at 0.
+            item_report.runs_ungraded = item_report.runs
+            print(f"[agentic] {item.id} UNGRADED: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"
             item_report.runs = 0

packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py

@@ -0,0 +1,62 @@
+# (C) 2026 GoodData Corporation. All rights reserved.
+"""Per-run failure records for the agentic evaluators.
+
+``core/runner.py`` records one of these for every failing run of a single-shot item. The
+agentic kinds cannot reuse it: each of them drives its own K-loop inside the evaluator and
+hands ``cli/agentic_runner`` a single aggregate per item, so by the time the runner sees
+the result the individual attempts are already collapsed into ``best``. The runs themselves
+are not lost -- every kind keeps its ``run_results`` list -- they simply never leave the
+evaluator. This turns that list into the same records the single-shot path writes.
+
+Keys mirror ``core.runner._failed_run_record`` so one consumer can read ``failed_runs`` from
+either path without branching on test kind. ``tool_call_count``/``tool_names`` are the
+addition: the agentic kinds capture tool calls per run, and an agent that produced a final
+answer having made no tool call at all answered from the model's own knowledge rather than
+from the workspace -- a distinction no other recorded field exposes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+
+def build_failed_runs(
+    run_results: Sequence[Any],
+    *,
+    passed: Callable[[Any], bool],
+    detail: Callable[[Any], dict],
+) -> list[dict]:
+    """One record per run that did not pass, in the order the runs happened.
+
+    ``passed`` and ``detail`` are supplied by the caller because neither is uniform across
+    kinds: guardrail reads ``run.passed`` while visualization reads
+    ``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything
+    read here directly goes through ``getattr`` with a default, so a kind whose run result
+    lacks a field (no response id, no tool capture) records a null instead of raising.
+
+    A run the judge could not grade is recorded like any other non-passing run, with its
+    ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the
+    verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``.
+    """
+    records: list[dict] = []
+    for run_index, run in enumerate(run_results, start=1):
+        if passed(run):
+            continue
+        tool_calls = list(getattr(run, "tool_call_events", None) or [])
+        reasoning_steps = list(getattr(run, "reasoning_steps", None) or [])
+        records.append(
+            {
+                "run_index": run_index,
+                "passed": False,
+                "error": getattr(run, "judge_error", None),
+                "detail": detail(run),
+                "conversation_id": getattr(run, "conversation_id", None),
+                "response_id": getattr(run, "response_id", None),
+                "reasoning_step_count": len(reasoning_steps),
+                "reasoning_steps": reasoning_steps,
+                "tool_call_count": len(tool_calls),
+                "tool_names": [getattr(e, "function_name", None) for e in tool_calls],
+            }
+        )
+    return records

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -12,6 +12,7 @@
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -792,6 +793,28 @@ class AlertSkillAssertionError(AgenticAssertionError):
     """Raised when an alert-skill evaluation fails."""
 
 
+def _run_detail(run: AlertRunResult) -> dict:
+    """The diagnostic fields for ONE run, shared by the best run and every failing one.
+
+    Extracted so a failing run is described by exactly the same keys as the winning run --
+    for this kind that means the full per-field check breakdown and the arguments the run sent.
+    """
+    ev = run.eval
+    return {
+        "alert_created": ev.alert_created,
+        "operator_correct": ev.operator_correct,
+        "threshold_correct": ev.threshold_correct,
+        "trigger_correct": ev.trigger_correct,
+        "filters_correct": ev.filters_correct,
+        "metric_correct": ev.metric_correct,
+        "recipients_correct": ev.recipients_correct,
+        "attributes_correct": ev.attributes_correct,
+        "granularity_correct": ev.granularity_correct,
+        "actual_alert_arguments": run.actual_alert_arguments,
+        "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events),
+    }
+
+
 def evaluate_agentic_alert_skill(
     host: str,
     token: str,
@@ -897,19 +920,14 @@ def _write_scores(ctx: RunTraceContext) -> None:
 
     best = summary.best
     ev = best.eval
-    detail = {
-        "alert_created": ev.alert_created,
-        "operator_correct": ev.operator_correct,
-        "threshold_correct": ev.threshold_correct,
-        "trigger_correct": ev.trigger_correct,
-        "filters_correct": ev.filters_correct,
-        "metric_correct": ev.metric_correct,
-        "recipients_correct": ev.recipients_correct,
-        "attributes_correct": ev.attributes_correct,
-        "granularity_correct": ev.granularity_correct,
-        "actual_alert_arguments": best.actual_alert_arguments,
-        "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
-    }
+    detail = _run_detail(best)
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.eval.strict_pass,
+        detail=_run_detail,
+    )
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
         gate_note = gate_failure_note(gate, runs_passed, runs_effective)
@@ -929,6 +947,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
+        exc.failed_runs = failed_runs
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -937,4 +956,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        # Also on the success path: pass@K clears the gate with one passing run, so a
+        # 1/3 item reports success while two of its runs failed for reasons worth reading.
+        failed_runs=failed_runs,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -6,6 +6,7 @@
 import time
 from dataclasses import dataclass, field
 
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -214,6 +215,44 @@ class GeneralQuestionAssertionError(AgenticAssertionError):
     """Raised when a general-question evaluation fails."""
 
 
+def _attach_diagnostics(
+    error: AgenticAssertionError | JudgeResponseError,
+    best: GeneralQuestionResult,
+    detail: dict,
+    failed_runs: list[dict],
+    runs_passed: int,
+    runs_effective: int,
+) -> None:
+    """Hang the report payload on a raised error, whichever error it is.
+
+    Both exits carry the same payload: the gate failure below, and the all-ungraded
+    JudgeResponseError above. They differ only in what the runner does with it -- the
+    second also marks the item errored -- so the attributes are set in one place.
+    """
+    error.reasoning_steps = best.reasoning_steps
+    error.conversation_id = best.conversation_id
+    error.response_id = best.response_id
+    error.detail = detail
+    error.runs_passed = runs_passed
+    error.runs_effective = runs_effective
+    error.failed_runs = failed_runs
+
+
+def _run_detail(run: GeneralQuestionResult) -> dict:
+    """The diagnostic fields for ONE run, shared by the best run and every failing one.
+
+    Extracted so a failing run is described by exactly the same keys as the winning run --
+    the two were worth comparing side by side, which they are not if only one of them
+    carries the judge's reasoning.
+    """
+    return {
+        "judge_passed": run.passed,
+        "judge_reasoning": run.reasoning,
+        "actual_output": run.actual_output,
+        "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events),
+    }
+
+
 def evaluate_agentic_general_question(
     host: str,
     token: str,
@@ -308,45 +347,51 @@ def _write_scores(ctx: RunTraceContext) -> None:
     item_timings = sum_timings([r.timings for r in summary.run_results])
     unscored = summary.judge_errors
 
-    if not summary.scored_run_results:
-        # Not one run produced a readable verdict, so this item has no result -- an error,
-        # not K failures. Raised after the trace link is queued so whatever the agent did
-        # is still linked, and carrying the timings so the runner can report what the item
-        # cost before it became unevaluable.
-        exc = JudgeResponseError(
-            f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): "
-            + " | ".join(unscored)
-        )
-        exc.timings = item_timings
-        raise exc
-
+    # Computed BEFORE the all-ungraded raise below, not after. An item whose every run went
+    # ungraded is the one where these matter most -- the judge broke, and the conversation
+    # ids are what someone needs to go read what the agent actually said -- but the raise
+    # used to happen first, so the runner caught a bare error and reported runs=0 with no
+    # records at all. `summary.best` already falls back to the unscored runs, and
+    # `runs_passed` is then 0, so nothing here needs a graded run to exist.
     runs_passed = sum(1 for r in summary.scored_run_results if r.passed)
     runs_effective = len(summary.run_results)
 
     best = summary.best
     detail = {
-        "judge_passed": best.passed,
-        "judge_reasoning": best.reasoning,
-        "actual_output": best.actual_output,
-        "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
+        **_run_detail(best),
         # Only present when it happened, so the usual JSON shape is unchanged. A
         # pass@K computed over fewer runs than --runs asked for is a weaker result and
         # the report has to say so.
         **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}),
     }
+    # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as
+    # non-passing here so it is recorded with its judge_error rather than silently dropped.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.judge_error is None and r.passed,
+        detail=_run_detail,
+    )
+
+    if not summary.scored_run_results:
+        # Not one run produced a readable verdict, so this item has no result -- an error,
+        # not K failures. Raised after the trace link is queued so whatever the agent did
+        # is still linked, and carrying the timings so the runner can report what the item
+        # cost before it became unevaluable.
+        error = JudgeResponseError(
+            f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): "
+            + " | ".join(unscored)
+        )
+        _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective)
+        error.timings = item_timings
+        raise error
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
         gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored))
         exc = GeneralQuestionAssertionError(
             f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}"
         )
-        exc.reasoning_steps = best.reasoning_steps
-        exc.conversation_id = best.conversation_id
-        exc.response_id = best.response_id
+        _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective)
         exc.timings = item_timings
-        exc.detail = detail
-        exc.runs_passed = runs_passed
-        exc.runs_effective = runs_effective
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -356,4 +401,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         response_id=best.response_id,
         detail=detail,
         timings=item_timings,
+        # Also on the success path: pass@K clears the gate with one passing run, so a
+        # 1/3 item reports success while two of its runs failed for reasons worth reading.
+        failed_runs=failed_runs,
     )