mirror of
https://github.com/docker/docker-agent.git
synced 2026-11-04 10:02:15 +00:00
221 lines
8.4 KiB
Go
221 lines
8.4 KiB
Go
package codingharness
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/docker/docker-agent/pkg/config/latest"
|
|
"github.com/docker/docker-agent/pkg/harness"
|
|
)
|
|
|
|
func TestCodexLargeJSONEvents(t *testing.T) {
|
|
for _, size := range []int{10000, 64 * 1024, 1024 * 1024} {
|
|
t.Run(strconv.Itoa(size), func(t *testing.T) {
|
|
text := strings.Repeat("界", size)
|
|
item, err := json.Marshal(map[string]any{
|
|
"type": "item.completed",
|
|
"item": map[string]any{"type": "agent_message", "text": text},
|
|
})
|
|
require.NoError(t, err)
|
|
useCodexCLI(t, string(item)+"\n"+codexCompleted, 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
var received strings.Builder
|
|
err = p.Run(ctx, "test", func(ev harness.Event) {
|
|
if ev.Type == harness.EventText {
|
|
received.WriteString(ev.Text)
|
|
}
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, text, received.String())
|
|
})
|
|
}
|
|
}
|
|
|
|
const codexCompleted = `{"type":"turn.completed","usage":{"input_tokens":10,"output_tokens":2}}` + "\n"
|
|
|
|
func TestCodexTurnOutcomes(t *testing.T) {
|
|
for _, tt := range []struct {
|
|
name string
|
|
output string
|
|
exit int
|
|
want string
|
|
}{
|
|
{name: "success", output: codexCompleted},
|
|
{name: "reconnected", output: `{"type":"error","message":"Reconnecting... 1/5"}` + "\n" + codexCompleted},
|
|
{name: "unknown event", output: `{"type":"future.event","message":{"detail":"ok"},"error":[1,2]}` + "\n" + codexCompleted},
|
|
{name: "whitespace and unterminated line", output: "\r\n \t" + strings.TrimSpace(codexCompleted)},
|
|
{name: "failed with zero exit", output: `{"type":"turn.failed","error":{"message":"quota exceeded"}}` + "\n", want: "codex turn failed"},
|
|
{name: "failed with nonzero exit", output: `{"type":"turn.failed","error":{"message":"quota exceeded"}}` + "\n", exit: 1, want: "codex turn failed"},
|
|
{name: "error then EOF", output: `{"type":"error","message":"connection lost"}` + "\n", want: "without turn.completed"},
|
|
{name: "missing terminal event", output: `{"type":"turn.started"}` + "\n", want: "without turn.completed"},
|
|
{name: "empty output", want: "without turn.completed"},
|
|
{name: "invalid JSON", output: "{broken\n" + codexCompleted, want: "invalid Codex JSON event"},
|
|
{name: "invalid envelope", output: "[]\n" + codexCompleted, want: "invalid Codex JSON event"},
|
|
{name: "missing type", output: "{}\n" + codexCompleted, want: "missing event type"},
|
|
{name: "nonzero exit after completion", output: codexCompleted, exit: 2, want: "exit status 2"},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
useCodexCLI(t, tt.output, tt.exit)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
err = p.Run(ctx, "test", func(harness.Event) {})
|
|
if tt.want == "" {
|
|
require.NoError(t, err)
|
|
} else {
|
|
require.ErrorContains(t, err, tt.want)
|
|
}
|
|
assert.NoError(t, ctx.Err(), "runner must finish without the watchdog")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCodexResumeArguments(t *testing.T) {
|
|
argsPath := useCodexCLI(t, codexCompleted, 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex, Model: "gpt-model"})
|
|
require.NoError(t, err)
|
|
require.NoError(t, p.Resume(t.Context(), "thread-id", "don't interpolate $(echo unsafe)", func(harness.Event) {}))
|
|
args, err := os.ReadFile(argsPath)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "exec\nresume\nthread-id\n--json\n--dangerously-bypass-approvals-and-sandbox\n-m\ngpt-model\ndon't interpolate $(echo unsafe)\n", string(args))
|
|
require.ErrorContains(t, p.Resume(t.Context(), "", "test", func(harness.Event) {}), "session ID is empty")
|
|
}
|
|
|
|
func TestCodexOversizedEventReturnsWithoutHanging(t *testing.T) {
|
|
useCodexCLI(t, `{"type":"item.completed","item":{"type":"agent_message","text":"`+strings.Repeat("x", 17*1024*1024)+`"}}`+"\n", 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
err = p.Run(ctx, "test", func(harness.Event) {})
|
|
require.ErrorContains(t, err, "stream line exceeds")
|
|
assert.NoError(t, ctx.Err(), "size rejection must not wait for the subprocess")
|
|
}
|
|
|
|
func useCodexCLI(t *testing.T, output string, exit int) string {
|
|
t.Helper()
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("shell script shim test")
|
|
}
|
|
dir := t.TempDir()
|
|
outPath := filepath.Join(dir, "events.jsonl")
|
|
argsPath := filepath.Join(dir, "args")
|
|
require.NoError(t, os.WriteFile(outPath, []byte(output), 0o600))
|
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "exit-code"), []byte(strconv.Itoa(exit)+"\n"), 0o600))
|
|
// Reuse the executable inode to avoid macOS's per-file cold-exec checks.
|
|
binary, err := filepath.Abs("testdata/codex")
|
|
require.NoError(t, err)
|
|
require.NoError(t, os.Symlink(binary, filepath.Join(dir, "codex")))
|
|
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
return argsPath
|
|
}
|
|
|
|
func TestCodexCancellation(t *testing.T) {
|
|
useCodexCLI(t, "", 0)
|
|
dir := strings.Split(os.Getenv("PATH"), string(os.PathListSeparator))[0]
|
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "wait"), nil, 0o600))
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
watchdog, stop := context.WithTimeout(ctx, 30*time.Second)
|
|
defer stop()
|
|
err = p.Run(watchdog, "test", func(ev harness.Event) {
|
|
if ev.Type == harness.EventSessionID {
|
|
cancel()
|
|
}
|
|
})
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
}
|
|
|
|
func TestCodexSlowConsumer(t *testing.T) {
|
|
useCodexCLI(t, `{"type":"item.completed","item":{"type":"agent_message","text":"answer"}}`+"\n"+codexCompleted, 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
err = p.Run(ctx, "test", func(ev harness.Event) {
|
|
if ev.Type == harness.EventText {
|
|
// The leader exits while the consumer is still handling this event.
|
|
<-time.After(2200 * time.Millisecond)
|
|
}
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestCodexLargeToolOutput(t *testing.T) {
|
|
output := strings.Repeat("output\n", 20000)
|
|
item, err := json.Marshal(map[string]any{
|
|
"type": "item.completed",
|
|
"item": map[string]any{"type": "command_execution", "id": "tool-1", "aggregated_output": output, "exit_code": 0},
|
|
})
|
|
require.NoError(t, err)
|
|
useCodexCLI(t, string(item)+"\n"+codexCompleted, 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
var result harness.Event
|
|
require.NoError(t, p.Run(ctx, "test", func(ev harness.Event) {
|
|
if ev.Type == harness.EventToolResult {
|
|
result = ev
|
|
}
|
|
}))
|
|
assert.Equal(t, "tool-1", result.ToolID)
|
|
assert.Equal(t, output, result.ToolOutput)
|
|
assert.False(t, result.ToolError)
|
|
}
|
|
|
|
func TestCodexValidationPerInvocation(t *testing.T) {
|
|
for _, resume := range []bool{false, true} {
|
|
t.Run(strconv.FormatBool(resume), func(t *testing.T) {
|
|
useCodexCLI(t, codexCompleted, 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
run := func(ctx context.Context) error {
|
|
if resume {
|
|
return p.Resume(ctx, "thread-id", "test", func(harness.Event) {})
|
|
}
|
|
return p.Run(ctx, "test", func(harness.Event) {})
|
|
}
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
defer cancel()
|
|
require.NoError(t, run(ctx))
|
|
dir := strings.Split(os.Getenv("PATH"), string(os.PathListSeparator))[0]
|
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "events.jsonl"), []byte(`{"type":"turn.started"}`+"\n"), 0o600))
|
|
require.ErrorContains(t, run(ctx), "without turn.completed")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCodexErrorsOmitStreamContents(t *testing.T) {
|
|
for _, output := range []string{
|
|
`{"type":"turn.failed","error":{"message":"private-output-marker"}}` + "\n",
|
|
`{"type":"error","message":"private-output-marker"}` + "\n",
|
|
`{"type":"private-output-marker` + "\n",
|
|
} {
|
|
useCodexCLI(t, output, 0)
|
|
p, err := Factory(&latest.HarnessConfig{Type: TypeCodex})
|
|
require.NoError(t, err)
|
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
|
|
err = p.Run(ctx, "test", func(harness.Event) {})
|
|
cancel()
|
|
require.Error(t, err)
|
|
assert.NotContains(t, err.Error(), "private-output-marker")
|
|
}
|
|
}
|