From 59bb1afa28042865a246dc6683d2dfbb6dda5888 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Sat, 25 Apr 2026 17:08:03 +0200 Subject: [PATCH] docs(cache): document response cache and harden file write durability Add a "Response Cache" section to the agent configuration reference and strengthen the file-backed cache so writes are not just atomic but also durable. - docs/configuration/agents/index.md: extend the schema example and property table with the new `cache` field, and add a dedicated "Response Cache" section describing the matching options (case_sensitive, trim_spaces), the storage backends (in-memory vs JSON file), and the atomicity guarantees of the file-backed store. - pkg/cache: fsync the temp file before rename and fsync the parent directory after rename so the rename itself is persisted across an OS crash. Concurrent readers are still guaranteed to see either the previous or the new content in full, never a torn write. - pkg/cache: add tests asserting that the success path leaves no leftover temp files and that a parallel reader never observes a partially written cache file under sustained concurrent stores. Assisted-By: docker-agent --- docs/configuration/agents/index.md | 41 +++++++++++++++++++ pkg/cache/cache.go | 41 +++++++++++++++++-- pkg/cache/cache_test.go | 63 ++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 3 deletions(-) diff --git a/docs/configuration/agents/index.md b/docs/configuration/agents/index.md index 1adb72117..b5ee2b394 100644 --- a/docs/configuration/agents/index.md +++ b/docs/configuration/agents/index.md @@ -48,6 +48,11 @@ agents: structured_output: # Optional: constrain output format name: string schema: object + cache: # Optional: response cache (skip the model on repeat questions) + enabled: boolean + case_sensitive: boolean + trim_spaces: boolean + path: string ```
@@ -83,6 +88,7 @@ agents: | `handoffs` | array | ✗ | List of agent names this agent can hand off the conversation to. Enables the `handoff` tool. See [Handoffs Routing]({{ '/concepts/multi-agent/#handoffs-routing' | relative_url }}). | | `hooks` | object | ✗ | Lifecycle hooks for running commands at various points. See [Hooks]({{ '/configuration/hooks/' | relative_url }}). | | `structured_output` | object | ✗ | Constrain agent output to match a JSON schema. See [Structured Output]({{ '/configuration/structured-output/' | relative_url }}). | +| `cache` | object | ✗ | Response cache. When the same user question is asked again, the previous answer is replayed verbatim and the model is not called. See [Response Cache](#response-cache) below. |
⚠️ max_iterations @@ -91,6 +97,41 @@ agents:
+## Response Cache + +The response cache short-circuits the model when the same user question is asked again. The first time a question is asked, the agent calls the model normally and stores the assistant's reply. Subsequent identical questions skip the model entirely and replay the stored reply verbatim. + +```yaml +agents: + root: + model: openai/gpt-5-mini + description: Cached assistant + instruction: You are a helpful assistant. + cache: + enabled: true # required to turn the cache on + case_sensitive: false # default: false ("Hello" == "hello") + trim_spaces: true # default: false (" hello " == "hello") + path: ./cache.json # optional: persist to disk; omit for in-memory +``` + +| Property | Type | Default | Description | +| ---------------- | ------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | boolean | `false` | Master switch. When `false` (or when the `cache` section is omitted), no caching is performed. | +| `case_sensitive` | boolean | `false` | When `true`, questions must match exactly (including case) to hit the cache. | +| `trim_spaces` | boolean | `false` | When `true`, leading and trailing whitespace is stripped from the question before it is compared. | +| `path` | string | _empty_ | When set, cache entries are persisted to a JSON file at the given path and reloaded on startup so the cache survives restarts. Relative paths resolve against the agent config directory. When empty, the cache lives in memory only. | + +**How it works** + +- The cache key is the latest user message in the session, normalized according to `case_sensitive` and `trim_spaces`. +- On a hit, the cached reply is added to the session as the assistant message and stop hooks fire normally — the rest of the agent (tools, sub-agents, the model) is bypassed. +- On a miss, the agent runs normally; the final assistant message produced by the first stop of the run is then stored under the question's key. +- Only the response to the original user question of a run is cached; follow-up turns inside the same `RunStream` are not. + +**File-backed storage** + +When `path` is set, every `Store` rewrites the entire cache file. Writes are **atomic**: the new content is written to a sibling temp file, `fsync`'d, and renamed over the destination, so a concurrent reader (or a process that crashes mid-write) will always see either the previous content or the new content in full — never a partially written file. The parent directory is also `fsync`'d after the rename so the rename itself is durable. + ## Welcome Message Display a message when users start a session: diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 956e9bc0a..2bbe9cb93 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -10,7 +10,11 @@ // - an in-memory map (the default), which keeps entries for the lifetime of // the process; // - a JSON-file backed store, which persists entries to disk so they -// survive restarts. +// survive restarts. Writes to the JSON file are atomic: the file is +// written to a sibling temp file, fsync'd, and renamed over the +// destination, so a concurrent reader (or a process that crashes +// mid-write) will always see either the previous content or the new +// content in full — never a partially written file. // // Two normalization options are exposed: // @@ -180,8 +184,18 @@ func (c *fileCache) Store(question, response string) { } // writeJSON atomically writes the given map to path as pretty-printed JSON. +// +// Atomicity is achieved by writing to a temporary file in the same +// directory and then renaming it over the destination: POSIX guarantees +// concurrent readers see either the old file or the new file in full, +// never a partially written one. +// +// Durability across an OS crash or power loss is achieved by fsync'ing +// the temp file before the rename and fsync'ing the parent directory +// after, so the rename itself is persisted. func writeJSON(path string, entries map[string]string) error { - if dir := filepath.Dir(path); dir != "" && dir != "." { + dir := filepath.Dir(path) + if dir != "" && dir != "." { if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("creating cache directory %q: %w", dir, err) } @@ -192,17 +206,27 @@ func writeJSON(path string, entries map[string]string) error { return fmt.Errorf("marshaling cache: %w", err) } - tmp, err := os.CreateTemp(filepath.Dir(path), ".cache-*.json") + tmp, err := os.CreateTemp(dir, ".cache-*.json") if err != nil { return fmt.Errorf("creating temp cache file: %w", err) } tmpName := tmp.Name() + // If anything below fails, make sure the temp file is cleaned up. + // On the success path the rename moves it away and Remove becomes a + // harmless no-op. defer os.Remove(tmpName) if _, err := tmp.Write(data); err != nil { tmp.Close() return fmt.Errorf("writing temp cache file: %w", err) } + // fsync the file contents to disk before the rename so a crash + // between rename and flush cannot leave the destination pointing at + // a zero-length file. + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("syncing temp cache file: %w", err) + } if err := tmp.Close(); err != nil { return fmt.Errorf("closing temp cache file: %w", err) } @@ -210,5 +234,16 @@ func writeJSON(path string, entries map[string]string) error { if err := os.Rename(tmpName, path); err != nil { return fmt.Errorf("renaming cache file: %w", err) } + + // fsync the parent directory so the rename itself is durable. + // Best-effort: directory fsync is not supported on every platform + // (e.g. Windows), and a failure here does not invalidate the data + // already written to disk. + if dirName := dir; dirName != "" { + if d, err := os.Open(dirName); err == nil { + _ = d.Sync() + _ = d.Close() + } + } return nil } diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 961d0c95d..acd1e6199 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -2,6 +2,8 @@ package cache import ( "encoding/json" + "errors" + "fmt" "os" "path/filepath" "testing" @@ -131,3 +133,64 @@ func TestFileCache_corruptFile(t *testing.T) { _, err := New(Config{Enabled: true, Path: path}) assert.Error(t, err) } + +// TestFileCache_atomicWriteLeavesNoTempFiles verifies that the rename-based +// atomic write does not leak temporary files on the happy path. +func TestFileCache_atomicWriteLeavesNoTempFiles(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cache.json") + + c, err := New(Config{Enabled: true, Path: path}) + require.NoError(t, err) + + for i := range 5 { + c.Store(fmt.Sprintf("q%d", i), fmt.Sprintf("a%d", i)) + } + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + + for _, e := range entries { + assert.Equal(t, "cache.json", e.Name(), + "unexpected leftover in cache directory: %q", e.Name()) + } +} + +// TestFileCache_concurrentStoreNeverYieldsTornFile verifies that concurrent +// Store calls always leave a fully valid JSON file behind — i.e. a parallel +// reader will never observe a half-written cache thanks to the +// rename-over-temp atomicity. +func TestFileCache_concurrentStoreNeverYieldsTornFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cache.json") + + c, err := New(Config{Enabled: true, Path: path}) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + defer close(done) + for i := range 50 { + c.Store(fmt.Sprintf("q%d", i), fmt.Sprintf("a%d", i)) + } + }() + + // While writes are happening, repeatedly read and parse the file. + // Without atomic rename, this would intermittently see truncated / + // half-written content and json.Unmarshal would error. + for range 100 { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + require.NoError(t, err) + if len(data) == 0 { + continue + } + var m map[string]string + require.NoError(t, json.Unmarshal(data, &m), + "reader observed a torn write: %q", string(data)) + } + + <-done +}