fix(eval): pin local agent image digest when building containers

docs/features/evaluation/index.md

@@ -340,7 +340,7 @@ evaluated agent run fails to authenticate.
 
 When `--base-image` is set, the eval harness builds a derived image on top of your base image at evaluation time. Two things happen automatically:
 
-1. **The docker-agent binary is injected** — by default it is copied from `docker/docker-agent:<version>`, pinned to this CLI's own release version, so eval results are reproducible against a known build rather than a moving target. A dev build (compiled from `main`, without a release version) falls back to `docker/docker-agent:edge`. This injection applies to every eval run, not just those using `--base-image`. Use `--agent-image <ref>` to inject a specific image instead — for example to pin CI to an older release, or to test against `docker/docker-agent:edge` deliberately. Pass `--agent-image none` to skip injection entirely and trust whatever `/docker-agent` binary is already present in your base image.
+1. **The docker-agent binary is injected** — by default it is copied from `docker/docker-agent:<version>`, pinned to this CLI's own release version, so eval results are reproducible against a known build rather than a moving target. A dev build (compiled from `main`, without a release version) falls back to `docker/docker-agent:edge`. When the selected image is available locally with a matching repository manifest digest, the build uses that digest so a locally rebuilt tag cannot resolve to an older registry image. Otherwise normal tag resolution applies. This injection applies to every eval run, not just those using `--base-image`. Use `--agent-image <ref>` to inject a specific image instead — for example to pin CI to an older release, or to test against `docker/docker-agent:edge` deliberately. Pass `--agent-image none` to skip injection entirely and trust whatever `/docker-agent` binary is already present in your base image.
 2. **The entrypoint is overridden** — Docker Agent replaces your base image's entrypoint with its own `/run.sh` wrapper.
 
 Your base image therefore only needs to provide the runtime environment: language runtimes, installed dependencies, test fixtures, the appropriate working directory, and so on. Any `ENTRYPOINT` or `CMD` defined in your base image is ignored.

pkg/evaluation/build.go

@@ -4,15 +4,19 @@ import (
 	"bytes"
 	"context"
 	_ "embed"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
+	"log/slog"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"text/template"
 
+	"github.com/google/go-containerregistry/pkg/name"
+
 	"github.com/docker/docker-agent/pkg/session"
 )
 
@@ -88,7 +92,6 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria
 		BaseImage      string
 		AgentImage     string
 	}
-	data.AgentImage = ResolvedAgentImage(r.Config)
 
 	if evals.WorkingDir == "" {
 		buildContext = r.EvalsDir
@@ -101,6 +104,8 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria
 		data.CopyWorkingDir = true
 	}
 
+	data.AgentImage = r.localAgentImage(ctx, ResolvedAgentImage(r.Config))
+
 	// Choose template based on whether a custom base image is provided
 	tmpl := dockerfileTemplate
 	if baseImage := r.resolveBaseImage(evals); baseImage != "" {
@@ -128,3 +133,31 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria
 
 	return strings.TrimSpace(string(output)), nil
 }
+
+// Pin a locally available manifest so BuildKit cannot resolve a rebuilt tag to an older registry image.
+func (r *Runner) localAgentImage(ctx context.Context, image string) string {
+	if image == "" || strings.Contains(image, "@") {
+		return image
+	}
+	ref, err := name.ParseReference(image)
+	if err != nil {
+		return image
+	}
+	cmd := exec.CommandContext(ctx, r.containerRuntimeOrDefault(), "image", "inspect", "--format", "{{json .RepoDigests}}", image)
+	output, err := cmd.Output()
+	if err != nil {
+		return image
+	}
+	var digests []string
+	if json.Unmarshal(output, &digests) != nil {
+		return image
+	}
+	for _, candidate := range digests {
+		digest, err := name.NewDigest(candidate)
+		if err == nil && digest.Context().Name() == ref.Context().Name() {
+			slog.DebugContext(ctx, "Pinning local eval agent image", "image", image, "digest", candidate)
+			return candidate
+		}
+	}
+	return image
+}

pkg/evaluation/build_test.go

@@ -0,0 +1,82 @@
+package evaluation
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/docker/docker-agent/pkg/config"
+	"github.com/docker/docker-agent/pkg/session"
+)
+
+func TestBuildEvalImagePinsLocalAgentDigest(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("the fake runtime is a POSIX shell script")
+	}
+	t.Parallel()
+
+	digest := "sha256:" + strings.Repeat("a", 64)
+	for _, tc := range []struct {
+		name, image, inspect, want string
+		failedInspect              bool
+	}{
+		{"local manifest", "docker/docker-agent:edge", `["docker/docker-agent@` + digest + `"]`, "docker/docker-agent@" + digest, false},
+		{"normalized repository", "docker/docker-agent:edge", `["docker.io/docker/docker-agent@` + digest + `"]`, "docker.io/docker/docker-agent@" + digest, false},
+		{"unrelated first digest", "docker/docker-agent:edge", `["other/agent@` + digest + `","docker/docker-agent@` + digest + `"]`, "docker/docker-agent@" + digest, false},
+		{"explicit digest", "docker/docker-agent@" + digest, "", "docker/docker-agent@" + digest, false},
+		{"skip injection", NoAgentImage, "", "", false},
+		{"missing local image", "docker/docker-agent:edge", "", "docker/docker-agent:edge", true},
+		{"null inspect", "docker/docker-agent:edge", "null", "docker/docker-agent:edge", false},
+		{"invalid then valid digest", "docker/docker-agent:edge", `["docker/docker-agent@bad","docker/docker-agent@` + digest + `"]`, "docker/docker-agent@" + digest, false},
+		{"default image", "", `["` + strings.Split(DefaultAgentImage(), ":")[0] + `@` + digest + `"]`, strings.Split(DefaultAgentImage(), ":")[0] + "@" + digest, false},
+		{"no manifest digest", "docker/docker-agent:edge", "[]", "docker/docker-agent:edge", false},
+		{"invalid inspect output", "docker/docker-agent:edge", "invalid", "docker/docker-agent:edge", false},
+		{"different repository", "docker/docker-agent:edge", `["other/agent@` + digest + `"]`, "docker/docker-agent:edge", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			dir := t.TempDir()
+			argsFile := filepath.Join(dir, "inspect-args")
+			dockerfile := filepath.Join(dir, "Dockerfile")
+			fakeRuntime := filepath.Join(dir, "runtime")
+			script := "#!/bin/sh\nif [ \"$1\" = image ]; then\n" +
+				" printf '%s\\n' \"$@\" > '" + argsFile + "'\n"
+			if tc.failedInspect {
+				script += " exit 1\n"
+			} else {
+				script += " printf '%s\\n' '" + tc.inspect + "'\n"
+			}
+			script += "else\n cat > '" + dockerfile + "'\n printf '%s\\n' sha256:eval-image\nfi\n"
+			require.NoError(t, os.WriteFile(fakeRuntime, []byte(script), 0o755))
+			runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), nil,
+				Config{EvalsDir: dir, ContainerRuntime: fakeRuntime, AgentImage: tc.image})
+			image, err := runner.buildEvalImage(t.Context(), &session.EvalCriteria{})
+			require.NoError(t, err)
+			assert.Equal(t, "sha256:eval-image", image)
+			content, err := os.ReadFile(dockerfile)
+			require.NoError(t, err)
+			if tc.want == "" {
+				assert.NotContains(t, string(content), "COPY --from=")
+			} else {
+				assert.Contains(t, string(content), "COPY --from="+tc.want+" /docker-agent /")
+			}
+			if tc.image == NoAgentImage || strings.Contains(tc.image, "@") {
+				_, err := os.Stat(argsFile)
+				assert.True(t, os.IsNotExist(err), "skip inspect for pinned or disabled injection")
+			} else {
+				args, err := os.ReadFile(argsFile)
+				require.NoError(t, err)
+				expected := tc.image
+				if expected == "" {
+					expected = DefaultAgentImage()
+				}
+				assert.Equal(t, "image\ninspect\n--format\n{{json .RepoDigests}}\n"+expected+"\n", string(args))
+			}
+		})
+	}
+}