mirror of
https://github.com/docker/docker-agent.git
synced 2026-11-04 10:02:15 +00:00
melmennaoui
54e78f0868
Add a code-based assertion evaluator and integrate it into the eval pipeline so assertions declared in the eval JSON are executed after the agent run, alongside the existing size/tool-call/relevance checks. New files: - assertions.go: runAssertions() evaluates 8 assertion types (contains, not_contains, equals, starts_with, ends_with, regex, cost_threshold, tool_called) against the agent response, cost, and tool-call list. - assertions_test.go: comprehensive tests for all assertion types. Modified files: - types.go: AssertionResults/AssertionsPassed/AssertionsTotal fields on Result; checkResults() reports assertion successes and failures. - eval.go: runSingleEval() runs assertions after relevance checks. - save.go: populateEvalResult() populates AssertionsCheck on the session's EvalResult. - progress.go: tracks and displays assertion failure counts in the live progress bar. - eval_test.go: checkResults tests for assertion pass/fail/zero. Closes #4086
89 lines
2.5 KiB
Go
89 lines
2.5 KiB
Go
package evaluation
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/docker/docker-agent/pkg/session"
|
|
)
|
|
|
|
// AssertionResult records the outcome of a single assertion evaluation.
|
|
type AssertionResult struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
Passed bool `json:"passed"`
|
|
Reason string `json:"reason,omitempty"`
|
|
}
|
|
|
|
// runAssertions evaluates all assertions against the agent response and
|
|
// auxiliary data (cost, tool calls).
|
|
func runAssertions(assertions []session.Assertion, response string, cost float64, toolCalls []string) []AssertionResult {
|
|
results := make([]AssertionResult, len(assertions))
|
|
for i, a := range assertions {
|
|
results[i] = evalAssertion(a, response, cost, toolCalls)
|
|
}
|
|
return results
|
|
}
|
|
|
|
func evalAssertion(a session.Assertion, response string, cost float64, toolCalls []string) AssertionResult {
|
|
r := AssertionResult{Name: a.Name, Type: a.Type}
|
|
switch strings.ToLower(a.Type) {
|
|
case "contains":
|
|
r.Passed = strings.Contains(response, a.Value)
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("response does not contain %q", a.Value)
|
|
}
|
|
case "not_contains":
|
|
r.Passed = !strings.Contains(response, a.Value)
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("response contains %q", a.Value)
|
|
}
|
|
case "equals":
|
|
r.Passed = strings.TrimSpace(response) == strings.TrimSpace(a.Value)
|
|
if !r.Passed {
|
|
r.Reason = "response does not equal expected value"
|
|
}
|
|
case "starts_with":
|
|
r.Passed = strings.HasPrefix(response, a.Value)
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("response does not start with %q", a.Value)
|
|
}
|
|
case "ends_with":
|
|
r.Passed = strings.HasSuffix(strings.TrimSpace(response), a.Value)
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("response does not end with %q", a.Value)
|
|
}
|
|
case "regex":
|
|
re, err := regexp.Compile(a.Value)
|
|
if err != nil {
|
|
r.Reason = fmt.Sprintf("invalid regex %q: %v", a.Value, err)
|
|
} else {
|
|
r.Passed = re.MatchString(response)
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("response does not match pattern %q", a.Value)
|
|
}
|
|
}
|
|
case "cost_threshold":
|
|
threshold, err := strconv.ParseFloat(a.Value, 64)
|
|
if err != nil {
|
|
r.Reason = fmt.Sprintf("invalid cost threshold %q: %v", a.Value, err)
|
|
} else {
|
|
r.Passed = cost <= threshold
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("cost $%.6f exceeds threshold $%.6f", cost, threshold)
|
|
}
|
|
}
|
|
case "tool_called":
|
|
r.Passed = slices.Contains(toolCalls, a.Value)
|
|
if !r.Passed {
|
|
r.Reason = fmt.Sprintf("tool %q was not called", a.Value)
|
|
}
|
|
default:
|
|
r.Reason = fmt.Sprintf("unknown assertion type %q", a.Type)
|
|
}
|
|
return r
|
|
}
|