feat(eval): report repeat stability and cost outliers

docs/features/evaluation/index.md

@@ -190,7 +190,41 @@ regular summary:
    pass^5: 66.7% (passed every time)
 ```
 
-These metrics are also included in the JSON results (`repeat_metrics`) and,
+The summary also groups repetitions by source eval file and prints:
+
+- **Stability counts and a per-eval table** — pass counts (`3/5`), execution
+  error counts, and status: `stable` (all pass), `flaky` (mixed pass/fail),
+  `failing` (none pass), or `incomplete` (fewer than `k` observed results).
+  Flaky evals appear first, followed by failing, incomplete, and stable evals.
+- **Failed repetition titles** — identify the individual runs to inspect in
+  the earlier output or saved sessions.
+- **Cost spread** — median, minimum, maximum, and sample count per eval,
+  excluding repetitions with execution errors. Failed quality checks still
+  contribute cost samples.
+- **High-cost outliers** — repetitions costing strictly more than **2× their
+  eval's median**, when there are at least three non-error samples and the
+  median is positive. This is a heuristic, not a statistical significance test;
+  zero or unreported prices do not trigger outlier flags.
+
+```console
+  Stability: 1 stable, 1 flaky, 1 failing, 0 incomplete
+  EVAL           STATUS   PASSED  ERRORS  COST MEDIAN [MIN, MAX]
+  evals/b.json   flaky    3/5     0       $0.010000 [$0.010000, $0.090000] (n=5)
+  evals/c.json   failing  0/5     5       n/a
+  evals/a.json   stable   5/5     0       $0.020000 [$0.018000, $0.022000] (n=5)
+  Failed repetitions (evals/b.json): B #2, B #4
+  Failed repetitions (evals/c.json): C #1, C #2, C #3, C #4, C #5
+
+  Cost outliers (>2x per-eval median; at least 3 non-error runs):
+   B #4 (evals/b.json): $0.090000 (9.0x median)
+```
+
+Per-eval diagnostics are included in the JSON results under
+`summary.repeat_metrics.evals`. They are informational only: they do not change
+pass/fail rules or the regression gate. Individual timing distributions are
+not reported; `Total Time` remains the run's wall-clock duration.
+
+Repeat metrics are also included in the JSON results (`repeat_metrics`) and,
 when comparing against a `--baseline`, are reported as informational deltas
 (see [Regression gate](#regression-gate)).
 

pkg/evaluation/repeat.go

@@ -0,0 +1,141 @@
+package evaluation
+
+import (
+	"cmp"
+	"fmt"
+	"io"
+	"slices"
+	"strings"
+	"text/tabwriter"
+)
+
+func computeRepeatDiagnostics(results []Result, k int) []RepeatEvalMetrics {
+	byEval := make(map[string][]Result)
+	for _, r := range results {
+		// Cancelled runs leave zero-value slots for evaluations never started.
+		if r.InputPath == "" {
+			continue
+		}
+		byEval[r.InputPath] = append(byEval[r.InputPath], r)
+	}
+
+	var evals []RepeatEvalMetrics
+	for path, runs := range byEval {
+		m := RepeatEvalMetrics{InputPath: path, Total: len(runs)}
+		var costs []Result
+		for _, r := range runs {
+			_, failures := r.checkResults()
+			if len(failures) == 0 {
+				m.Passed++
+			} else {
+				m.FailedRuns = append(m.FailedRuns, cmp.Or(r.Title, path))
+			}
+			if r.Error != "" {
+				m.Errors++
+				continue
+			}
+			costs = append(costs, r)
+		}
+		slices.Sort(m.FailedRuns)
+		m.Cost = computeRepeatCost(costs)
+		switch {
+		case m.Total < k:
+			m.Status = "incomplete"
+		case m.Passed == m.Total:
+			m.Status = "stable"
+		case m.Passed == 0:
+			m.Status = "failing"
+		default:
+			m.Status = "flaky"
+		}
+		evals = append(evals, m)
+	}
+
+	slices.SortFunc(evals, func(a, b RepeatEvalMetrics) int {
+		return cmp.Or(cmp.Compare(repeatStatusRank(a.Status), repeatStatusRank(b.Status)), cmp.Compare(a.InputPath, b.InputPath))
+	})
+	return evals
+}
+
+func repeatStatusRank(status string) int {
+	switch status {
+	case "flaky":
+		return 0
+	case "failing":
+		return 1
+	case "incomplete":
+		return 2
+	default:
+		return 3
+	}
+}
+
+func computeRepeatCost(runs []Result) *RepeatCostMetrics {
+	if len(runs) == 0 {
+		return nil
+	}
+	slices.SortFunc(runs, func(a, b Result) int {
+		return cmp.Or(cmp.Compare(a.Cost, b.Cost), cmp.Compare(a.Title, b.Title))
+	})
+	mid := len(runs) / 2
+	median := runs[mid].Cost
+	if len(runs)%2 == 0 {
+		median = runs[mid-1].Cost/2 + median/2
+	}
+	m := &RepeatCostMetrics{
+		Samples: len(runs),
+		Median:  median,
+		Min:     runs[0].Cost,
+		Max:     runs[len(runs)-1].Cost,
+	}
+	// Avoid labelling one of two observations, or unpriced runs, as outliers.
+	if len(runs) >= 3 && median > 0 {
+		for _, r := range runs {
+			if r.Cost > 2*median {
+				m.Outliers = append(m.Outliers, RepeatCostOutlier{Title: r.Title, Cost: r.Cost})
+			}
+		}
+	}
+	return m
+}
+
+func printRepeatDiagnostics(out io.Writer, evals []RepeatEvalMetrics) {
+	if len(evals) == 0 {
+		return
+	}
+	counts := make(map[string]int)
+	for _, e := range evals {
+		counts[e.Status]++
+	}
+	fmt.Fprintf(out, "\n  Stability: %d stable, %d flaky, %d failing, %d incomplete\n", counts["stable"], counts["flaky"], counts["failing"], counts["incomplete"])
+	tw := tabwriter.NewWriter(out, 2, 8, 2, ' ', 0)
+	fmt.Fprintln(tw, "  EVAL\tSTATUS\tPASSED\tERRORS\tCOST MEDIAN [MIN, MAX]\t")
+	for _, e := range evals {
+		cost := "n/a"
+		if e.Cost != nil {
+			cost = fmt.Sprintf("$%.6f [$%.6f, $%.6f] (n=%d)", e.Cost.Median, e.Cost.Min, e.Cost.Max, e.Cost.Samples)
+		}
+		fmt.Fprintf(tw, "  %s\t%s\t%d/%d\t%d\t%s\t\n", e.InputPath, e.Status, e.Passed, e.Total, e.Errors, cost)
+	}
+	_ = tw.Flush()
+
+	for _, e := range evals {
+		if len(e.FailedRuns) > 0 {
+			fmt.Fprintf(out, "  Failed repetitions (%s): %s\n", e.InputPath, strings.Join(e.FailedRuns, ", "))
+		}
+	}
+
+	hasOutliers := false
+	for _, e := range evals {
+		if e.Cost == nil {
+			continue
+		}
+		for _, o := range e.Cost.Outliers {
+			if !hasOutliers {
+				fmt.Fprintln(out, "\n  Cost outliers (>2x per-eval median; at least 3 non-error runs):")
+				hasOutliers = true
+			}
+			fmt.Fprintf(out, "   %s (%s): $%.6f (%.1fx median)\n", o.Title, e.InputPath, o.Cost, o.Cost/e.Cost.Median)
+		}
+	}
+}

pkg/evaluation/repeat_test.go

@@ -0,0 +1,194 @@
+package evaluation
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComputeRepeatDiagnostics(t *testing.T) {
+	t.Parallel()
+	results := []Result{
+		{InputPath: "stable.json", Title: "Same title #1", Cost: 0.01},
+		{InputPath: "stable.json", Title: "Same title #2", Cost: 0.01},
+		{InputPath: "stable.json", Title: "Same title #3", Cost: 0.01},
+		{InputPath: "flaky.json", Title: "Same title #1", Cost: 0.01},
+		{InputPath: "flaky.json", Title: "Same title #2", Cost: 0.02, SizeExpected: "M", Size: "S"},
+		{InputPath: "flaky.json", Title: "Same title #3", Cost: 0.09},
+		{InputPath: "failing.json", Title: "Fails #1", Error: "boom"},
+		{InputPath: "failing.json", Title: "Fails #2", Error: "boom"},
+		{InputPath: "failing.json", Title: "Fails #3", Error: "boom"},
+		{InputPath: "partial.json", Title: "Partial #1"},
+		{},
+	}
+	original := slices.Clone(results)
+	evals := computeRepeatDiagnostics(results, 3)
+	require.Len(t, evals, 4)
+	assert.Equal(t, original, results, "diagnostics must not reorder or modify results")
+
+	flaky := evals[0]
+	assert.Equal(t, "flaky.json", flaky.InputPath)
+	assert.Equal(t, "flaky", flaky.Status)
+	assert.Equal(t, 2, flaky.Passed)
+	assert.Equal(t, 3, flaky.Total)
+	assert.Zero(t, flaky.Errors)
+	assert.Equal(t, []string{"Same title #2"}, flaky.FailedRuns)
+	require.NotNil(t, flaky.Cost)
+	assert.Equal(t, 3, flaky.Cost.Samples)
+	assert.InDelta(t, 0.02, flaky.Cost.Median, 1e-9)
+	assert.InDelta(t, 0.01, flaky.Cost.Min, 1e-9)
+	assert.InDelta(t, 0.09, flaky.Cost.Max, 1e-9)
+	assert.Equal(t, []RepeatCostOutlier{{Title: "Same title #3", Cost: 0.09}}, flaky.Cost.Outliers)
+
+	failing := evals[1]
+	assert.Equal(t, "failing", failing.Status)
+	assert.Zero(t, failing.Passed)
+	assert.Equal(t, 3, failing.Errors)
+	assert.Nil(t, failing.Cost)
+	assert.Equal(t, []string{"Fails #1", "Fails #2", "Fails #3"}, failing.FailedRuns)
+
+	partial := evals[2]
+	assert.Equal(t, "incomplete", partial.Status)
+	assert.Equal(t, 1, partial.Passed)
+	assert.Equal(t, 1, partial.Total)
+	assert.Equal(t, "stable", evals[3].Status)
+
+	slices.Reverse(results)
+	assert.Equal(t, evals, computeRepeatDiagnostics(results, 3), "output must be deterministic")
+}
+
+func TestComputeRepeatCost(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name    string
+		costs   []float64
+		median  float64
+		outlier bool
+	}{
+		{name: "one sample", costs: []float64{0.01}, median: 0.01},
+		{name: "two samples", costs: []float64{0.01, 0.09}, median: 0.05},
+		{name: "even median", costs: []float64{0.01, 0.02, 0.03, 0.09}, median: 0.025, outlier: true},
+		{name: "at threshold", costs: []float64{0.01, 0.01, 0.02}, median: 0.01},
+		{name: "above threshold", costs: []float64{0.01, 0.01, 0.021}, median: 0.01, outlier: true},
+		{name: "zero costs", costs: []float64{0, 0, 0}, median: 0},
+		{name: "zero median", costs: []float64{0, 0, 0.09}, median: 0},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var runs []Result
+			for _, cost := range tc.costs {
+				runs = append(runs, Result{Cost: cost})
+			}
+			m := computeRepeatCost(runs)
+			require.NotNil(t, m)
+			assert.InDelta(t, tc.median, m.Median, 1e-9)
+			assert.Equal(t, len(tc.costs), m.Samples)
+			assert.Equal(t, tc.outlier, len(m.Outliers) > 0)
+		})
+	}
+	assert.Nil(t, computeRepeatCost(nil))
+}
+
+func TestRepeatDiagnosticsExecutionErrors(t *testing.T) {
+	t.Parallel()
+	m := computeRepeatMetrics([]Result{
+		{InputPath: "a.json", Title: "A #1", Cost: 0.01},
+		{InputPath: "a.json", Title: "A #2", Cost: 100, Error: "judge failed"},
+		{InputPath: "a.json", Title: "A #3", Cost: 0.01},
+	}, 3)
+	require.NotNil(t, m)
+	require.Len(t, m.Evals, 1)
+	e := m.Evals[0]
+	assert.Equal(t, "flaky", e.Status)
+	assert.Equal(t, 2, e.Passed)
+	assert.Equal(t, 1, e.Errors)
+	assert.Equal(t, []string{"A #2"}, e.FailedRuns)
+	require.NotNil(t, e.Cost)
+	assert.Equal(t, 2, e.Cost.Samples)
+	assert.InDelta(t, 0.01, e.Cost.Max, 1e-9)
+	assert.Empty(t, e.Cost.Outliers)
+}
+
+func TestRepeatDiagnosticsOrdering(t *testing.T) {
+	t.Parallel()
+	evals := computeRepeatDiagnostics([]Result{
+		{InputPath: "z.json"},
+		{InputPath: "a.json"},
+		{InputPath: "z.json"},
+		{InputPath: "a.json"},
+	}, 2)
+	require.Len(t, evals, 2)
+	assert.Equal(t, "a.json", evals[0].InputPath)
+	assert.Equal(t, "z.json", evals[1].InputPath)
+	assert.Empty(t, computeRepeatDiagnostics([]Result{{}}, 3))
+}
+
+func TestPrintRepeatDiagnostics(t *testing.T) {
+	t.Parallel()
+	results := []Result{
+		{InputPath: "a.json", Title: "A #1", Cost: 0.01},
+		{InputPath: "a.json", Title: "A #2", Cost: 0.01, SizeExpected: "M", Size: "S"},
+		{InputPath: "a.json", Title: "A #3", Cost: 0.09},
+		{InputPath: "b.json", Title: "B #1", Error: "boom"},
+		{InputPath: "b.json", Title: "B #2", Error: "boom"},
+		{InputPath: "b.json", Title: "B #3", Error: "boom"},
+	}
+	summary := computeSummary(results)
+	summary.RepeatMetrics = computeRepeatMetrics(results, 3)
+	var buf bytes.Buffer
+	printSummary(&buf, summary, time.Minute)
+	output := buf.String()
+	assert.Contains(t, output, "pass@3: 50.0%")
+	assert.Contains(t, output, "pass^3: 0.0%")
+	assert.Contains(t, output, "Stability: 0 stable, 1 flaky, 1 failing, 0 incomplete")
+	assert.Contains(t, output, "2/3")
+	assert.Contains(t, output, "$0.010000 [$0.010000, $0.090000] (n=3)")
+	assert.Contains(t, output, "n/a")
+	assert.Contains(t, output, "Failed repetitions (a.json): A #2")
+	assert.Contains(t, output, "Cost outliers (>2x per-eval median; at least 3 non-error runs):")
+	assert.Contains(t, output, "A #3 (a.json): $0.090000 (9.0x median)")
+	assert.Contains(t, output, "Total Cost: $0.110000")
+}
+
+func TestPrintRepeatDiagnosticsNoOutliers(t *testing.T) {
+	t.Parallel()
+	var buf bytes.Buffer
+	printRepeatDiagnostics(&buf, computeRepeatDiagnostics([]Result{
+		{InputPath: "a.json", Cost: 0.01},
+		{InputPath: "a.json", Cost: 0.01},
+	}, 2))
+	assert.Contains(t, buf.String(), "1 stable, 0 flaky, 0 failing, 0 incomplete")
+	assert.NotContains(t, buf.String(), "Cost outliers")
+	assert.NotContains(t, buf.String(), "Failed repetitions")
+
+	buf.Reset()
+	printRepeatDiagnostics(&buf, nil)
+	assert.Empty(t, buf.String())
+}
+
+func TestRepeatDiagnosticsSavedJSON(t *testing.T) {
+	t.Parallel()
+	run := &EvalRun{
+		Name: "repeat-test",
+		Results: []Result{
+			{InputPath: "a.json", Title: "A #1"},
+			{InputPath: "a.json", Title: "A #2", Error: "boom"},
+		},
+	}
+	run.Summary = computeSummary(run.Results)
+	run.Summary.RepeatMetrics = computeRepeatMetrics(run.Results, 2)
+	path, err := SaveRunSessionsJSON(run, t.TempDir())
+	require.NoError(t, err)
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	var saved RunOutput
+	require.NoError(t, json.Unmarshal(data, &saved))
+	assert.Equal(t, run.Summary.RepeatMetrics, saved.Summary.RepeatMetrics)
+	assert.Contains(t, string(data), `"failed_runs"`)
+}

pkg/evaluation/scoring.go

@@ -114,6 +114,7 @@ func printSummary(out io.Writer, summary Summary, duration time.Duration) {
 		fmt.Fprintf(out, "\n  Repeat metrics (k=%d, %d unique evals):\n", rm.K, rm.Total)
 		fmt.Fprintf(out, "   pass@%d: %.1f%% (passed at least once)\n", rm.K, rm.PassK*100)
 		fmt.Fprintf(out, "   pass^%d: %.1f%% (passed every time)\n", rm.K, rm.HatK*100)
+		printRepeatDiagnostics(out, rm.Evals)
 	}
 
 	fmt.Fprintf(out, "\nTotal Cost: $%.6f\n", summary.TotalCost)
@@ -194,5 +195,6 @@ func computeRepeatMetrics(results []Result, k int) *RepeatMetrics {
 		PassK: float64(anyPass) / float64(total),
 		HatK:  float64(allPass) / float64(total),
 		Total: total,
+		Evals: computeRepeatDiagnostics(results, k),
 	}
 }

pkg/evaluation/types.go

@@ -200,10 +200,37 @@ const DefaultContainerRuntime = "docker"
 //   - pass^k: fraction of unique evaluations that passed every repetition
 //     ("all pass"). Measures determinism / reliability.
 type RepeatMetrics struct {
-	K     int     `json:"k"`
-	PassK float64 `json:"pass_at_k"`  // any-pass rate
-	HatK  float64 `json:"pass_hat_k"` // all-pass rate
-	Total int     `json:"total"`      // number of unique evaluations
+	K     int                 `json:"k"`
+	PassK float64             `json:"pass_at_k"`  // any-pass rate
+	HatK  float64             `json:"pass_hat_k"` // all-pass rate
+	Total int                 `json:"total"`      // number of unique evaluations
+	Evals []RepeatEvalMetrics `json:"evals,omitempty"`
+}
+
+// RepeatEvalMetrics describes the observed stability of one source evaluation.
+type RepeatEvalMetrics struct {
+	InputPath  string             `json:"input_path"`
+	Status     string             `json:"status"`
+	Passed     int                `json:"passed"`
+	Total      int                `json:"total"`
+	Errors     int                `json:"errors"`
+	FailedRuns []string           `json:"failed_runs,omitempty"`
+	Cost       *RepeatCostMetrics `json:"cost,omitempty"`
+}
+
+// RepeatCostMetrics summarizes costs from repetitions without execution errors.
+type RepeatCostMetrics struct {
+	Samples  int                 `json:"samples"`
+	Median   float64             `json:"median"`
+	Min      float64             `json:"min"`
+	Max      float64             `json:"max"`
+	Outliers []RepeatCostOutlier `json:"outliers,omitempty"`
+}
+
+// RepeatCostOutlier identifies a repetition costing more than twice its eval's median.
+type RepeatCostOutlier struct {
+	Title string  `json:"title"`
+	Cost  float64 `json:"cost"`
 }
 
 // containerRuntimeOrDefault returns the container runtime executable to