From f2c8671000fbdeedbee5fb6b4ba7d43974bc459c Mon Sep 17 00:00:00 2001 From: David Gageot Date: Sun, 4 Oct 2026 18:06:35 +0200 Subject: [PATCH] fix(eval): pin local agent image digest when building containers --- docs/features/evaluation/index.md | 2 +- pkg/evaluation/build.go | 35 ++++++++++++- pkg/evaluation/build_test.go | 82 +++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 pkg/evaluation/build_test.go diff --git a/docs/features/evaluation/index.md b/docs/features/evaluation/index.md index 6bd5030eb..67fec0177 100644 --- a/docs/features/evaluation/index.md +++ b/docs/features/evaluation/index.md @@ -340,7 +340,7 @@ evaluated agent run fails to authenticate. When `--base-image` is set, the eval harness builds a derived image on top of your base image at evaluation time. Two things happen automatically: -1. **The docker-agent binary is injected** — by default it is copied from `docker/docker-agent:`, pinned to this CLI's own release version, so eval results are reproducible against a known build rather than a moving target. A dev build (compiled from `main`, without a release version) falls back to `docker/docker-agent:edge`. This injection applies to every eval run, not just those using `--base-image`. Use `--agent-image ` to inject a specific image instead — for example to pin CI to an older release, or to test against `docker/docker-agent:edge` deliberately. Pass `--agent-image none` to skip injection entirely and trust whatever `/docker-agent` binary is already present in your base image. +1. **The docker-agent binary is injected** — by default it is copied from `docker/docker-agent:`, pinned to this CLI's own release version, so eval results are reproducible against a known build rather than a moving target. A dev build (compiled from `main`, without a release version) falls back to `docker/docker-agent:edge`. When the selected image is available locally with a matching repository manifest digest, the build uses that digest so a locally rebuilt tag cannot resolve to an older registry image. Otherwise normal tag resolution applies. This injection applies to every eval run, not just those using `--base-image`. Use `--agent-image ` to inject a specific image instead — for example to pin CI to an older release, or to test against `docker/docker-agent:edge` deliberately. Pass `--agent-image none` to skip injection entirely and trust whatever `/docker-agent` binary is already present in your base image. 2. **The entrypoint is overridden** — Docker Agent replaces your base image's entrypoint with its own `/run.sh` wrapper. Your base image therefore only needs to provide the runtime environment: language runtimes, installed dependencies, test fixtures, the appropriate working directory, and so on. Any `ENTRYPOINT` or `CMD` defined in your base image is ignored. diff --git a/pkg/evaluation/build.go b/pkg/evaluation/build.go index 0b0610836..9bbfd340a 100644 --- a/pkg/evaluation/build.go +++ b/pkg/evaluation/build.go @@ -4,15 +4,19 @@ import ( "bytes" "context" _ "embed" + "encoding/json" "errors" "fmt" "io/fs" + "log/slog" "os" "os/exec" "path/filepath" "strings" "text/template" + "github.com/google/go-containerregistry/pkg/name" + "github.com/docker/docker-agent/pkg/session" ) @@ -88,7 +92,6 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria BaseImage string AgentImage string } - data.AgentImage = ResolvedAgentImage(r.Config) if evals.WorkingDir == "" { buildContext = r.EvalsDir @@ -101,6 +104,8 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria data.CopyWorkingDir = true } + data.AgentImage = r.localAgentImage(ctx, ResolvedAgentImage(r.Config)) + // Choose template based on whether a custom base image is provided tmpl := dockerfileTemplate if baseImage := r.resolveBaseImage(evals); baseImage != "" { @@ -128,3 +133,31 @@ func (r *Runner) buildEvalImage(ctx context.Context, evals *session.EvalCriteria return strings.TrimSpace(string(output)), nil } + +// Pin a locally available manifest so BuildKit cannot resolve a rebuilt tag to an older registry image. +func (r *Runner) localAgentImage(ctx context.Context, image string) string { + if image == "" || strings.Contains(image, "@") { + return image + } + ref, err := name.ParseReference(image) + if err != nil { + return image + } + cmd := exec.CommandContext(ctx, r.containerRuntimeOrDefault(), "image", "inspect", "--format", "{{json .RepoDigests}}", image) + output, err := cmd.Output() + if err != nil { + return image + } + var digests []string + if json.Unmarshal(output, &digests) != nil { + return image + } + for _, candidate := range digests { + digest, err := name.NewDigest(candidate) + if err == nil && digest.Context().Name() == ref.Context().Name() { + slog.DebugContext(ctx, "Pinning local eval agent image", "image", image, "digest", candidate) + return candidate + } + } + return image +} diff --git a/pkg/evaluation/build_test.go b/pkg/evaluation/build_test.go new file mode 100644 index 000000000..5b7b4dc7c --- /dev/null +++ b/pkg/evaluation/build_test.go @@ -0,0 +1,82 @@ +package evaluation + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/docker/docker-agent/pkg/config" + "github.com/docker/docker-agent/pkg/session" +) + +func TestBuildEvalImagePinsLocalAgentDigest(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the fake runtime is a POSIX shell script") + } + t.Parallel() + + digest := "sha256:" + strings.Repeat("a", 64) + for _, tc := range []struct { + name, image, inspect, want string + failedInspect bool + }{ + {"local manifest", "docker/docker-agent:edge", `["docker/docker-agent@` + digest + `"]`, "docker/docker-agent@" + digest, false}, + {"normalized repository", "docker/docker-agent:edge", `["docker.io/docker/docker-agent@` + digest + `"]`, "docker.io/docker/docker-agent@" + digest, false}, + {"unrelated first digest", "docker/docker-agent:edge", `["other/agent@` + digest + `","docker/docker-agent@` + digest + `"]`, "docker/docker-agent@" + digest, false}, + {"explicit digest", "docker/docker-agent@" + digest, "", "docker/docker-agent@" + digest, false}, + {"skip injection", NoAgentImage, "", "", false}, + {"missing local image", "docker/docker-agent:edge", "", "docker/docker-agent:edge", true}, + {"null inspect", "docker/docker-agent:edge", "null", "docker/docker-agent:edge", false}, + {"invalid then valid digest", "docker/docker-agent:edge", `["docker/docker-agent@bad","docker/docker-agent@` + digest + `"]`, "docker/docker-agent@" + digest, false}, + {"default image", "", `["` + strings.Split(DefaultAgentImage(), ":")[0] + `@` + digest + `"]`, strings.Split(DefaultAgentImage(), ":")[0] + "@" + digest, false}, + {"no manifest digest", "docker/docker-agent:edge", "[]", "docker/docker-agent:edge", false}, + {"invalid inspect output", "docker/docker-agent:edge", "invalid", "docker/docker-agent:edge", false}, + {"different repository", "docker/docker-agent:edge", `["other/agent@` + digest + `"]`, "docker/docker-agent:edge", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + argsFile := filepath.Join(dir, "inspect-args") + dockerfile := filepath.Join(dir, "Dockerfile") + fakeRuntime := filepath.Join(dir, "runtime") + script := "#!/bin/sh\nif [ \"$1\" = image ]; then\n" + + " printf '%s\\n' \"$@\" > '" + argsFile + "'\n" + if tc.failedInspect { + script += " exit 1\n" + } else { + script += " printf '%s\\n' '" + tc.inspect + "'\n" + } + script += "else\n cat > '" + dockerfile + "'\n printf '%s\\n' sha256:eval-image\nfi\n" + require.NoError(t, os.WriteFile(fakeRuntime, []byte(script), 0o755)) + runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), nil, + Config{EvalsDir: dir, ContainerRuntime: fakeRuntime, AgentImage: tc.image}) + image, err := runner.buildEvalImage(t.Context(), &session.EvalCriteria{}) + require.NoError(t, err) + assert.Equal(t, "sha256:eval-image", image) + content, err := os.ReadFile(dockerfile) + require.NoError(t, err) + if tc.want == "" { + assert.NotContains(t, string(content), "COPY --from=") + } else { + assert.Contains(t, string(content), "COPY --from="+tc.want+" /docker-agent /") + } + if tc.image == NoAgentImage || strings.Contains(tc.image, "@") { + _, err := os.Stat(argsFile) + assert.True(t, os.IsNotExist(err), "skip inspect for pinned or disabled injection") + } else { + args, err := os.ReadFile(argsFile) + require.NoError(t, err) + expected := tc.image + if expected == "" { + expected = DefaultAgentImage() + } + assert.Equal(t, "image\ninspect\n--format\n{{json .RepoDigests}}\n"+expected+"\n", string(args)) + } + }) + } +}