nprasad2077/docker-agent
1
1 Fork 0
mirror of https://github.com/docker/docker-agent.git synced 2026-11-04 10:02:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(evaluation): reuse an immutable runtime launcher

Browse Source 
Assisted-By: docker-agent
Signed-off-by: David Gageot <david.gageot@docker.com>
This commit is contained in:
David Gageot committed 2026-10-17 16:32:22 +02:00
1 parent 300533f245
commit 3a2d99a548
7 files changed
+72 -27
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/evaluation/build_test.go
+7 -11
View File
@@ -18,16 +18,12 @@ func TestBuildEvalImagePinsLocalAgentDigest(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake runtime is a POSIX shell script")
}
// Write before parallel tests can fork and inherit a writer to the executable,
// which would make exec fail with ETXTBSY. Subtests only symlink this script.
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
require.NoError(t, os.WriteFile(sharedRuntime, []byte(
"#!/bin/sh\nd=${0%/*}\n"+
"if [ \"$1\" = image ]; then\n"+
" printf '%s\\n' \"$@\" > \"$d/inspect-args\"\n"+
" if [ -f \"$d/inspect-fail\" ]; then exit 1; fi\n"+
" cat \"$d/inspect-output\"\n"+
"else\n cat > \"$d/Dockerfile\"\n printf '%s\\n' sha256:eval-image\nfi\n"), 0o755))
const script = "#!/bin/sh\nd=${0%/*}\n" +
"if [ \"$1\" = image ]; then\n" +
" printf '%s\\n' \"$@\" > \"$d/inspect-args\"\n" +
" if [ -f \"$d/inspect-fail\" ]; then exit 1; fi\n" +
" cat \"$d/inspect-output\"\n" +
"else\n cat > \"$d/Dockerfile\"\n printf '%s\\n' sha256:eval-image\nfi\n"
t.Parallel()
digest := "sha256:" + strings.Repeat("a", 64)
@@ -58,7 +54,7 @@ func TestBuildEvalImagePinsLocalAgentDigest(t *testing.T) {
if tc.failedInspect {
require.NoError(t, os.WriteFile(filepath.Join(dir, "inspect-fail"), nil, 0o644))
}
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))
writeMockCLI(t, fakeRuntime, script)
runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), nil,
Config{EvalsDir: dir, ContainerRuntime: fakeRuntime, AgentImage: tc.image})
image, err := runner.buildEvalImage(t.Context(), &session.EvalCriteria{})
pkg/evaluation/eval_judge_test.go
+1 -3
View File
@@ -24,8 +24,6 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"hello"}`)
t.Parallel()
for _, tt := range []struct {
@@ -62,7 +60,7 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) {
require.NoError(t, os.Mkdir(evalsDir, 0o700))
require.NoError(t, os.WriteFile(filepath.Join(evalsDir, "session.json"), []byte(`{"evals":{`+tt.criteria+`}}`), 0o600))
fakeRuntime := filepath.Join(dir, "runtime")
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))
writeFakeContainerRuntime(t, fakeRuntime, `{"type":"agent_choice","content":"hello"}`)
run, err := Evaluate(t.Context(), &bytes.Buffer{}, &bytes.Buffer{}, false, "test",
&config.RuntimeConfig{EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"TYPESAFE_API_KEY": "secret"})},
Config{AgentFilename: agentPath, EvalsDir: evalsDir, JudgeType: JudgeTypeEvaluator, JudgeModel: "relevance", Concurrency: 1, ContainerRuntime: fakeRuntime})
pkg/evaluation/eval_test.go
+4 -6
View File
@@ -77,8 +77,6 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime executable is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"ok"}`)
t.Parallel()
for _, tt := range []struct {
@@ -115,7 +113,7 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) {
}
fakeRuntime := filepath.Join(tmpDir, "fake-runtime")
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))
writeFakeContainerRuntime(t, fakeRuntime, `{"type":"agent_choice","content":"ok"}`)
runConfig := &config.RuntimeConfig{
EnvProviderForTests: environment.NewNoEnvProvider(),
@@ -1146,7 +1144,7 @@ func TestRunDockerAgentInContainerCancelInterruptsDocker(t *testing.T) {
binDir := filepath.Join(tmpDir, "bin")
require.NoError(t, os.Mkdir(binDir, 0o755))
script := "#!/bin/sh\nexec \"" + testBin + "\" -test.run='^TestRunDockerAgentInContainerHelperProcess$'\n"
require.NoError(t, os.WriteFile(filepath.Join(binDir, "docker"), []byte(script), 0o755))
writeMockCLI(t, filepath.Join(binDir, "docker"), script)
t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("GO_WANT_HELPER_PROCESS", "1")
@@ -1228,11 +1226,11 @@ func TestContainerRuntimeOrDefault(t *testing.T) {
assert.Equal(t, "podman", custom.containerRuntimeOrDefault())
}
// Write before t.Parallel so forked children cannot inherit a writable executable.
// Keep each script body beside its launcher symlink.
func writeFakeContainerRuntime(t *testing.T, path, output string) {
t.Helper()
script := "#!/bin/sh\necho \"$@\" > \"${0%/*}/args\"\necho '" + output + "'\n"
require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
writeMockCLI(t, path, script)
}
// TestRunDockerAgentInContainerUsesConfiguredRuntime proves that container
pkg/evaluation/flavors_test.go
+2 -3
View File
@@ -18,8 +18,7 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime executable is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
require.NoError(t, os.WriteFile(sharedRuntime, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"${0%/*}/args\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"), 0o755))
const script = "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"${0%/*}/args\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"
t.Parallel()
for _, setup := range []string{"", "echo setup"} {
@@ -28,7 +27,7 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) {
dir := t.TempDir()
argsFile := filepath.Join(dir, "args")
fake := filepath.Join(dir, "runtime")
require.NoError(t, os.Symlink(sharedRuntime, fake))
writeMockCLI(t, fake, script)
runConfig := &config.RuntimeConfig{
Config: config.Config{Flavors: []string{"baseline", "fixtures,with spaces"}},
EnvProviderForTests: environment.NewNoEnvProvider(),
pkg/evaluation/judge_container_test.go
+3 -4
View File
@@ -108,8 +108,7 @@ func TestContainerMountsJudgeProviderWithoutCredentials(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
require.NoError(t, os.WriteFile(sharedRuntime, []byte(`#!/bin/sh
const script = `#!/bin/sh
printf '%s\n' "$@" > "${0%/*}/args"
for arg in "$@"; do
case "$arg" in
@@ -122,7 +121,7 @@ for arg in "$@"; do
done
printf '%s' "${HOST_JUDGE_KEY-}" > "${0%/*}/host-key"
echo '{"type":"agent_choice","content":"hello"}'
`), 0o755))
`
t.Parallel()
for _, setup := range []string{"", "echo setup"} {
@@ -130,7 +129,7 @@ echo '{"type":"agent_choice","content":"hello"}'
t.Parallel()
dir := t.TempDir()
fakeRuntime := filepath.Join(dir, "runtime")
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))
writeMockCLI(t, fakeRuntime, script)
runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), &config.RuntimeConfig{
Config: config.Config{Providers: map[string]latest.ProviderConfig{"corporate": {Provider: "typesafe", TokenKey: "HOST_JUDGE_KEY"}}},
EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"HOST_JUDGE_KEY": "secret-value"}),
pkg/evaluation/mock_script_test.go
+53
View File
@@ -0,0 +1,53 @@
package evaluation
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Capture the fixture path before tests change the process working directory.
var mockCLILauncher, mockCLILauncherErr = filepath.Abs("testdata/cli")
func writeMockCLI(t *testing.T, path, script string) {
t.Helper()
require.NoError(t, mockCLILauncherErr)
require.NoError(t, os.WriteFile(path+".script", []byte(script), 0o600))
require.NoError(t, os.Symlink(mockCLILauncher, path))
}
func TestMockCLI(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("POSIX shell fixture")
}
dir := filepath.Join(t.TempDir(), "space ' $dir")
require.NoError(t, os.Mkdir(dir, 0o700))
first := filepath.Join(dir, "docker")
second := filepath.Join(dir, "sbx")
writeMockCLI(t, first, `printf '<%s>\n' "$@"; printf diagnostic >&2; exit 7`)
writeMockCLI(t, second, `exec /bin/sh -c 'printf alternate; exit 9'`)
a, err := os.Stat(first)
require.NoError(t, err)
b, err := os.Stat(second)
require.NoError(t, err)
assert.True(t, os.SameFile(a, b))
t.Chdir(t.TempDir())
cmd := exec.CommandContext(t.Context(), first, "", "two words", "$(echo literal)")
cmd.Env = []string{"PATH=" + dir}
out, err := cmd.CombinedOutput()
var exitErr *exec.ExitError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, 7, exitErr.ExitCode())
assert.Equal(t, "<>\n<two words>\n<$(echo literal)>\ndiagnostic", string(out))
cmd = exec.CommandContext(t.Context(), second)
cmd.Env = []string{"PATH=" + dir}
out, err = cmd.Output()
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, 9, exitErr.ExitCode())
assert.Equal(t, "alternate", string(out))
}
pkg/evaluation/testdata/cli
Vendored Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
. "$0.script"