From 3a2d99a548bdb6f1934835763d990aae337a5763 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Sat, 17 Oct 2026 13:57:43 +0200 Subject: [PATCH] test(evaluation): reuse an immutable runtime launcher Assisted-By: docker-agent Signed-off-by: David Gageot --- pkg/evaluation/build_test.go | 18 ++++----- pkg/evaluation/eval_judge_test.go | 4 +- pkg/evaluation/eval_test.go | 10 ++--- pkg/evaluation/flavors_test.go | 5 +-- pkg/evaluation/judge_container_test.go | 7 ++-- pkg/evaluation/mock_script_test.go | 53 ++++++++++++++++++++++++++ pkg/evaluation/testdata/cli | 2 + 7 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 pkg/evaluation/mock_script_test.go create mode 100755 pkg/evaluation/testdata/cli diff --git a/pkg/evaluation/build_test.go b/pkg/evaluation/build_test.go index b49956d38..c91874f5d 100644 --- a/pkg/evaluation/build_test.go +++ b/pkg/evaluation/build_test.go @@ -18,16 +18,12 @@ func TestBuildEvalImagePinsLocalAgentDigest(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake runtime is a POSIX shell script") } - // Write before parallel tests can fork and inherit a writer to the executable, - // which would make exec fail with ETXTBSY. Subtests only symlink this script. - sharedRuntime := filepath.Join(t.TempDir(), "runtime") - require.NoError(t, os.WriteFile(sharedRuntime, []byte( - "#!/bin/sh\nd=${0%/*}\n"+ - "if [ \"$1\" = image ]; then\n"+ - " printf '%s\\n' \"$@\" > \"$d/inspect-args\"\n"+ - " if [ -f \"$d/inspect-fail\" ]; then exit 1; fi\n"+ - " cat \"$d/inspect-output\"\n"+ - "else\n cat > \"$d/Dockerfile\"\n printf '%s\\n' sha256:eval-image\nfi\n"), 0o755)) + const script = "#!/bin/sh\nd=${0%/*}\n" + + "if [ \"$1\" = image ]; then\n" + + " printf '%s\\n' \"$@\" > \"$d/inspect-args\"\n" + + " if [ -f \"$d/inspect-fail\" ]; then exit 1; fi\n" + + " cat \"$d/inspect-output\"\n" + + "else\n cat > \"$d/Dockerfile\"\n printf '%s\\n' sha256:eval-image\nfi\n" t.Parallel() digest := "sha256:" + strings.Repeat("a", 64) @@ -58,7 +54,7 @@ func TestBuildEvalImagePinsLocalAgentDigest(t *testing.T) { if tc.failedInspect { require.NoError(t, os.WriteFile(filepath.Join(dir, "inspect-fail"), nil, 0o644)) } - require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) + writeMockCLI(t, fakeRuntime, script) runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), nil, Config{EvalsDir: dir, ContainerRuntime: fakeRuntime, AgentImage: tc.image}) image, err := runner.buildEvalImage(t.Context(), &session.EvalCriteria{}) diff --git a/pkg/evaluation/eval_judge_test.go b/pkg/evaluation/eval_judge_test.go index 0e0e9407d..ec454109a 100644 --- a/pkg/evaluation/eval_judge_test.go +++ b/pkg/evaluation/eval_judge_test.go @@ -24,8 +24,6 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime is a POSIX shell script") } - sharedRuntime := filepath.Join(t.TempDir(), "runtime") - writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"hello"}`) t.Parallel() for _, tt := range []struct { @@ -62,7 +60,7 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) { require.NoError(t, os.Mkdir(evalsDir, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(evalsDir, "session.json"), []byte(`{"evals":{`+tt.criteria+`}}`), 0o600)) fakeRuntime := filepath.Join(dir, "runtime") - require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) + writeFakeContainerRuntime(t, fakeRuntime, `{"type":"agent_choice","content":"hello"}`) run, err := Evaluate(t.Context(), &bytes.Buffer{}, &bytes.Buffer{}, false, "test", &config.RuntimeConfig{EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"TYPESAFE_API_KEY": "secret"})}, Config{AgentFilename: agentPath, EvalsDir: evalsDir, JudgeType: JudgeTypeEvaluator, JudgeModel: "relevance", Concurrency: 1, ContainerRuntime: fakeRuntime}) diff --git a/pkg/evaluation/eval_test.go b/pkg/evaluation/eval_test.go index 23f6d356e..b9d59403c 100644 --- a/pkg/evaluation/eval_test.go +++ b/pkg/evaluation/eval_test.go @@ -77,8 +77,6 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime executable is a POSIX shell script") } - sharedRuntime := filepath.Join(t.TempDir(), "runtime") - writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"ok"}`) t.Parallel() for _, tt := range []struct { @@ -115,7 +113,7 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) { } fakeRuntime := filepath.Join(tmpDir, "fake-runtime") - require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) + writeFakeContainerRuntime(t, fakeRuntime, `{"type":"agent_choice","content":"ok"}`) runConfig := &config.RuntimeConfig{ EnvProviderForTests: environment.NewNoEnvProvider(), @@ -1146,7 +1144,7 @@ func TestRunDockerAgentInContainerCancelInterruptsDocker(t *testing.T) { binDir := filepath.Join(tmpDir, "bin") require.NoError(t, os.Mkdir(binDir, 0o755)) script := "#!/bin/sh\nexec \"" + testBin + "\" -test.run='^TestRunDockerAgentInContainerHelperProcess$'\n" - require.NoError(t, os.WriteFile(filepath.Join(binDir, "docker"), []byte(script), 0o755)) + writeMockCLI(t, filepath.Join(binDir, "docker"), script) t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) t.Setenv("GO_WANT_HELPER_PROCESS", "1") @@ -1228,11 +1226,11 @@ func TestContainerRuntimeOrDefault(t *testing.T) { assert.Equal(t, "podman", custom.containerRuntimeOrDefault()) } -// Write before t.Parallel so forked children cannot inherit a writable executable. +// Keep each script body beside its launcher symlink. func writeFakeContainerRuntime(t *testing.T, path, output string) { t.Helper() script := "#!/bin/sh\necho \"$@\" > \"${0%/*}/args\"\necho '" + output + "'\n" - require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) + writeMockCLI(t, path, script) } // TestRunDockerAgentInContainerUsesConfiguredRuntime proves that container diff --git a/pkg/evaluation/flavors_test.go b/pkg/evaluation/flavors_test.go index 28d213646..6d6a306f1 100644 --- a/pkg/evaluation/flavors_test.go +++ b/pkg/evaluation/flavors_test.go @@ -18,8 +18,7 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime executable is a POSIX shell script") } - sharedRuntime := filepath.Join(t.TempDir(), "runtime") - require.NoError(t, os.WriteFile(sharedRuntime, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"${0%/*}/args\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"), 0o755)) + const script = "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"${0%/*}/args\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n" t.Parallel() for _, setup := range []string{"", "echo setup"} { @@ -28,7 +27,7 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) { dir := t.TempDir() argsFile := filepath.Join(dir, "args") fake := filepath.Join(dir, "runtime") - require.NoError(t, os.Symlink(sharedRuntime, fake)) + writeMockCLI(t, fake, script) runConfig := &config.RuntimeConfig{ Config: config.Config{Flavors: []string{"baseline", "fixtures,with spaces"}}, EnvProviderForTests: environment.NewNoEnvProvider(), diff --git a/pkg/evaluation/judge_container_test.go b/pkg/evaluation/judge_container_test.go index 1b67731c5..488e58d65 100644 --- a/pkg/evaluation/judge_container_test.go +++ b/pkg/evaluation/judge_container_test.go @@ -108,8 +108,7 @@ func TestContainerMountsJudgeProviderWithoutCredentials(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime is a POSIX shell script") } - sharedRuntime := filepath.Join(t.TempDir(), "runtime") - require.NoError(t, os.WriteFile(sharedRuntime, []byte(`#!/bin/sh + const script = `#!/bin/sh printf '%s\n' "$@" > "${0%/*}/args" for arg in "$@"; do case "$arg" in @@ -122,7 +121,7 @@ for arg in "$@"; do done printf '%s' "${HOST_JUDGE_KEY-}" > "${0%/*}/host-key" echo '{"type":"agent_choice","content":"hello"}' -`), 0o755)) +` t.Parallel() for _, setup := range []string{"", "echo setup"} { @@ -130,7 +129,7 @@ echo '{"type":"agent_choice","content":"hello"}' t.Parallel() dir := t.TempDir() fakeRuntime := filepath.Join(dir, "runtime") - require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) + writeMockCLI(t, fakeRuntime, script) runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), &config.RuntimeConfig{ Config: config.Config{Providers: map[string]latest.ProviderConfig{"corporate": {Provider: "typesafe", TokenKey: "HOST_JUDGE_KEY"}}}, EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"HOST_JUDGE_KEY": "secret-value"}), diff --git a/pkg/evaluation/mock_script_test.go b/pkg/evaluation/mock_script_test.go new file mode 100644 index 000000000..3c4cb4cdb --- /dev/null +++ b/pkg/evaluation/mock_script_test.go @@ -0,0 +1,53 @@ +package evaluation + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Capture the fixture path before tests change the process working directory. +var mockCLILauncher, mockCLILauncherErr = filepath.Abs("testdata/cli") + +func writeMockCLI(t *testing.T, path, script string) { + t.Helper() + require.NoError(t, mockCLILauncherErr) + require.NoError(t, os.WriteFile(path+".script", []byte(script), 0o600)) + require.NoError(t, os.Symlink(mockCLILauncher, path)) +} + +func TestMockCLI(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell fixture") + } + dir := filepath.Join(t.TempDir(), "space ' $dir") + require.NoError(t, os.Mkdir(dir, 0o700)) + first := filepath.Join(dir, "docker") + second := filepath.Join(dir, "sbx") + writeMockCLI(t, first, `printf '<%s>\n' "$@"; printf diagnostic >&2; exit 7`) + writeMockCLI(t, second, `exec /bin/sh -c 'printf alternate; exit 9'`) + a, err := os.Stat(first) + require.NoError(t, err) + b, err := os.Stat(second) + require.NoError(t, err) + assert.True(t, os.SameFile(a, b)) + t.Chdir(t.TempDir()) + cmd := exec.CommandContext(t.Context(), first, "", "two words", "$(echo literal)") + cmd.Env = []string{"PATH=" + dir} + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 7, exitErr.ExitCode()) + assert.Equal(t, "<>\n\n<$(echo literal)>\ndiagnostic", string(out)) + cmd = exec.CommandContext(t.Context(), second) + cmd.Env = []string{"PATH=" + dir} + out, err = cmd.Output() + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 9, exitErr.ExitCode()) + assert.Equal(t, "alternate", string(out)) +} diff --git a/pkg/evaluation/testdata/cli b/pkg/evaluation/testdata/cli new file mode 100755 index 000000000..c6afda2c9 --- /dev/null +++ b/pkg/evaluation/testdata/cli @@ -0,0 +1,2 @@ +#!/bin/sh +. "$0.script"