diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 906ba3041..4d880f25f 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -15,6 +15,11 @@ on: description: 'Skip build+smoke steps (faster iteration on lint/tests)' type: boolean default: false + soak_level: + description: 'Soak test level: full (quick+asan+nightly), quick (10min only), none' + type: choice + options: ['full', 'quick', 'none'] + default: 'quick' permissions: contents: read @@ -503,3 +508,88 @@ jobs: exit 1 } Write-Host "=== Windows Defender: clean ===" + + # ── Step 6: Soak tests (after smoke, per-platform) ────────── + soak-quick: + if: ${{ inputs.soak_level != 'none' && !inputs.skip_builds }} + needs: [smoke-unix] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + goos: linux + goarch: amd64 + cc: gcc + cxx: g++ + - os: ubuntu-24.04-arm + goos: linux + goarch: arm64 + cc: gcc + cxx: g++ + - os: macos-14 + goos: darwin + goarch: arm64 + cc: cc + cxx: c++ + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Install deps (Linux) + if: startsWith(matrix.os, 'ubuntu') + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git + + - name: Build (release mode) + run: scripts/build.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} + + - name: Quick soak (10 min) + run: scripts/soak-test.sh build/c/codebase-memory-mcp 10 + + - name: Upload soak metrics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: soak-quick-${{ matrix.goos }}-${{ matrix.goarch }} + path: soak-results/ + retention-days: 14 + + soak-asan: + if: ${{ inputs.soak_level == 'full' && !inputs.skip_builds }} + needs: [smoke-unix] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + goarch: amd64 + - os: ubuntu-24.04-arm + goarch: arm64 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git + + - name: Build (ASan + LeakSanitizer) + run: | + scripts/test.sh CC=gcc CXX=g++ + # test.sh builds with ASan; the test-runner binary has ASan linked + cp build/c/test-runner build/c/codebase-memory-mcp-asan || true + scripts/build.sh CC=gcc CXX=g++ + + - name: ASan soak (15 min) + env: + ASAN_OPTIONS: "detect_leaks=1:halt_on_error=0" + run: scripts/soak-test.sh build/c/codebase-memory-mcp 15 + + - name: Upload ASan soak metrics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: soak-asan-linux-${{ matrix.goarch }} + path: soak-results/ + retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index faffd8b76..d00089bca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,11 @@ on: required: false type: boolean default: false + soak_level: + description: 'Soak test level: full (quick+asan), quick (10min only), none (skip)' + type: choice + options: ['full', 'quick', 'none'] + default: 'quick' permissions: contents: write @@ -505,9 +510,84 @@ jobs: } Write-Host "=== Windows Defender: clean ===" - # ── Step 5: Create DRAFT release (not public yet) ───────────── + # ── Step 5a: Quick soak (parallel with smoke, per-platform) ──── + soak-quick: + if: ${{ inputs.soak_level != 'none' }} + needs: [build-unix] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + goos: linux + goarch: amd64 + cc: gcc + cxx: g++ + - os: ubuntu-24.04-arm + goos: linux + goarch: arm64 + cc: gcc + cxx: g++ + - os: macos-14 + goos: darwin + goarch: arm64 + cc: cc + cxx: c++ + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - name: Install deps (Linux) + if: startsWith(matrix.os, 'ubuntu') + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git + - name: Build (release mode) + run: scripts/build.sh --version ${{ inputs.version }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} + - name: Quick soak (10 min) + run: scripts/soak-test.sh build/c/codebase-memory-mcp 10 + - name: Upload soak metrics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: soak-quick-${{ matrix.goos }}-${{ matrix.goarch }} + path: soak-results/ + retention-days: 14 + + # ── Step 5b: ASan soak (Linux only, parallel) ───────────────── + soak-asan: + if: ${{ inputs.soak_level == 'full' }} + needs: [build-unix] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + goarch: amd64 + - os: ubuntu-24.04-arm + goarch: arm64 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git + - name: Build (release mode for soak) + run: scripts/build.sh CC=gcc CXX=g++ + - name: ASan soak (15 min) + env: + ASAN_OPTIONS: "detect_leaks=1:halt_on_error=0" + run: scripts/soak-test.sh build/c/codebase-memory-mcp 15 + - name: Upload ASan soak metrics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: soak-asan-linux-${{ matrix.goarch }} + path: soak-results/ + retention-days: 14 + + # ── Step 6: Create DRAFT release (not public yet) ───────────── release-draft: - needs: [smoke-unix, smoke-windows, security-static, codeql-gate] + needs: [smoke-unix, smoke-windows, security-static, codeql-gate, soak-quick] + if: ${{ !cancelled() && !failure() }} runs-on: ubuntu-latest permissions: contents: write diff --git a/Makefile.cbm b/Makefile.cbm index a5dc2120d..b3bb4a8c1 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -101,7 +101,8 @@ FOUNDATION_SRCS = \ src/foundation/compat_thread.c \ src/foundation/compat_fs.c \ src/foundation/compat_regex.c \ - src/foundation/mem.c + src/foundation/mem.c \ + src/foundation/diagnostics.c # Existing extraction C code (compiled from current location) EXTRACTION_SRCS = \ diff --git a/scripts/soak-test.sh b/scripts/soak-test.sh new file mode 100755 index 000000000..f7a5c33fb --- /dev/null +++ b/scripts/soak-test.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# soak-test.sh — Endurance test for codebase-memory-mcp. +# +# Runs compressed workload cycles: queries, file mutations, reindexes, idle periods. +# Reads diagnostics from /tmp/cbm-diagnostics-.json (requires CBM_DIAGNOSTICS=1). +# Outputs metrics to soak-results/ and exits 0 (pass) or 1 (fail). +# +# Usage: +# scripts/soak-test.sh [--skip-crash-test] +# +# Tiers: +# 10 min = quick soak (CI gate) +# 15 min = ASan soak (leak detection) +# 240 min = nightly (compressed 4h = ~5 days real usage) + +set -euo pipefail + +BINARY="${1:?Usage: soak-test.sh }" +DURATION_MIN="${2:?Usage: soak-test.sh }" +SKIP_CRASH="${3:-}" +BINARY=$(cd "$(dirname "$BINARY")" && pwd)/$(basename "$BINARY") + +RESULTS_DIR="soak-results" +mkdir -p "$RESULTS_DIR" + +METRICS_CSV="$RESULTS_DIR/metrics.csv" +LATENCY_CSV="$RESULTS_DIR/latency.csv" +SUMMARY="$RESULTS_DIR/summary.txt" + +echo "timestamp,uptime_s,rss_bytes,heap_committed,fd_count,query_count,query_max_us" > "$METRICS_CSV" +echo "timestamp,tool,duration_ms,exit_code" > "$LATENCY_CSV" +> "$SUMMARY" + +DURATION_S=$((DURATION_MIN * 60)) + +echo "=== soak-test: binary=$BINARY duration=${DURATION_MIN}m ===" + +# ── Helper: create test project ────────────────────────────────── + +SOAK_PROJECT=$(mktemp -d) +mkdir -p "$SOAK_PROJECT/src" "$SOAK_PROJECT/lib" + +# Create a realistic small project with multiple languages +cat > "$SOAK_PROJECT/src/main.py" << 'PYEOF' +from lib.utils import compute, validate + +def process(data): + if validate(data): + return compute(data) + return None + +def main(): + result = process({"key": "value"}) + print(result) + +if __name__ == "__main__": + main() +PYEOF + +cat > "$SOAK_PROJECT/lib/utils.py" << 'PYEOF' +def compute(data): + total = sum(len(str(v)) for v in data.values()) + return total + +def validate(data): + return isinstance(data, dict) and len(data) > 0 + +def helper(): + return 42 +PYEOF + +cat > "$SOAK_PROJECT/src/server.go" << 'GOEOF' +package main + +import "fmt" + +func StartServer(port int) error { + fmt.Printf("listening on %d\n", port) + return nil +} + +func HandleRequest(path string) string { + return "ok: " + path +} +GOEOF + +cat > "$SOAK_PROJECT/config.yaml" << 'YAMLEOF' +database: + host: localhost + port: 5432 +server: + workers: 4 +YAMLEOF + +# Init git repo (required for watcher) +git -C "$SOAK_PROJECT" init -q 2>/dev/null +git -C "$SOAK_PROJECT" add -A 2>/dev/null +git -C "$SOAK_PROJECT" -c user.email=test@test -c user.name=test commit -q -m "init" 2>/dev/null + +# ── Helper: run CLI tool call and record latency ───────────────── + +cli_call() { + local tool="$1" + local args="$2" + local t0 + t0=$(python3 -c "import time; print(int(time.time()*1000))") + "$BINARY" cli "$tool" "$args" > /dev/null 2>&1 + local rc=$? + local t1 + t1=$(python3 -c "import time; print(int(time.time()*1000))") + local dur=$((t1 - t0)) + echo "$(date +%s),$tool,$dur,$rc" >> "$LATENCY_CSV" +} + +# ── Helper: collect diagnostics snapshot ───────────────────────── + +collect_snapshot() { + local diag_file="/tmp/cbm-diagnostics-${SERVER_PID}.json" + if [ -f "$diag_file" ]; then + local ts=$(date +%s) + local uptime=$(python3 -c "import json; d=json.load(open('$diag_file')); print(d.get('uptime_s',0))" 2>/dev/null || echo "0") + local rss=$(python3 -c "import json; d=json.load(open('$diag_file')); print(d.get('rss_bytes',0))" 2>/dev/null || echo "0") + local commit=$(python3 -c "import json; d=json.load(open('$diag_file')); print(d.get('heap_committed_bytes',0))" 2>/dev/null || echo "0") + local fds=$(python3 -c "import json; d=json.load(open('$diag_file')); print(d.get('fd_count',0))" 2>/dev/null || echo "0") + local qcount=$(python3 -c "import json; d=json.load(open('$diag_file')); print(d.get('query_count',0))" 2>/dev/null || echo "0") + local qmax=$(python3 -c "import json; d=json.load(open('$diag_file')); print(d.get('query_max_us',0))" 2>/dev/null || echo "0") + echo "$ts,$uptime,$rss,$commit,$fds,$qcount,$qmax" >> "$METRICS_CSV" + fi +} + +# ── Phase 1: Start MCP server with diagnostics ────────────────── + +echo "--- Phase 1: start server ---" +# Keep stdin open with a sleeping process — kill it to send EOF +sleep 999999 | CBM_DIAGNOSTICS=1 "$BINARY" > /dev/null 2>"$RESULTS_DIR/server-stderr.log" & +SERVER_PID=$! +SLEEP_PID=$(jobs -p | head -1) +sleep 3 + +if ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo "FAIL: server did not start" + kill "$SLEEP_PID" 2>/dev/null || true + exit 1 +fi +echo "OK: server running (pid=$SERVER_PID)" + +# ── Phase 2: Initial index ─────────────────────────────────────── + +echo "--- Phase 2: initial index ---" +cli_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}" +sleep 6 # wait for diagnostics write +collect_snapshot + +# Derive project name (same logic as cbm_project_name_from_path) +PROJ_NAME=$(echo "$SOAK_PROJECT" | sed 's|^/||; s|/|-|g') + +BASELINE_RSS=$(python3 -c "import json; d=json.load(open('/tmp/cbm-diagnostics-${SERVER_PID}.json')); print(d.get('rss_bytes',0))" 2>/dev/null || echo "0") +BASELINE_FDS=$(python3 -c "import json; d=json.load(open('/tmp/cbm-diagnostics-${SERVER_PID}.json')); print(d.get('fd_count',0))" 2>/dev/null || echo "0") +echo "OK: baseline RSS=${BASELINE_RSS} FDs=${BASELINE_FDS}" + +# ── Phase 3: Compressed workload loop ──────────────────────────── + +echo "--- Phase 3: workload loop (${DURATION_MIN}m) ---" +START_TIME=$(date +%s) +END_TIME=$((START_TIME + DURATION_S)) +CYCLE=0 +LAST_MUTATE=0 +LAST_REINDEX=0 + +while [ "$(date +%s)" -lt "$END_TIME" ]; do + NOW=$(date +%s) + CYCLE=$((CYCLE + 1)) + + # Queries every 2 seconds + cli_call search_graph "{\"project\":\"$PROJ_NAME\",\"name_pattern\":\".*compute.*\"}" + cli_call trace_call_path "{\"project\":\"$PROJ_NAME\",\"function_name\":\"compute\",\"direction\":\"both\"}" + + # File mutation every 2 minutes + if [ $((NOW - LAST_MUTATE)) -ge 120 ]; then + echo "# mutation at cycle $CYCLE $(date)" >> "$SOAK_PROJECT/src/main.py" + git -C "$SOAK_PROJECT" add -A 2>/dev/null + git -C "$SOAK_PROJECT" -c user.email=test@test -c user.name=test commit -q -m "cycle $CYCLE" 2>/dev/null || true + LAST_MUTATE=$NOW + fi + + # Full reindex every 5 minutes + if [ $((NOW - LAST_REINDEX)) -ge 300 ]; then + cli_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}" + LAST_REINDEX=$NOW + fi + + # Collect diagnostics every 30 seconds + if [ $((CYCLE % 15)) -eq 0 ]; then + collect_snapshot + fi + + sleep 2 +done + +# ── Phase 4: Idle period + final snapshot ──────────────────────── + +echo "--- Phase 4: idle (30s) ---" +sleep 30 +collect_snapshot + +# Check idle CPU +IDLE_CPU=$(ps -o %cpu= -p "$SERVER_PID" 2>/dev/null | tr -d ' ' || echo "0") +echo "OK: idle CPU=${IDLE_CPU}%" + +# ── Phase 5: Crash recovery test ──────────────────────────────── + +if [ "$SKIP_CRASH" != "--skip-crash-test" ]; then + echo "--- Phase 5: crash recovery ---" + + # Start a reindex then kill -9 + cli_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}" & + INDEX_PID=$! + sleep 1 + kill -9 "$INDEX_PID" 2>/dev/null || true + wait "$INDEX_PID" 2>/dev/null || true + + # Verify the server is still alive (kill was on CLI, not server) + if kill -0 "$SERVER_PID" 2>/dev/null; then + echo "OK: server survived crash test" + + # Verify clean re-index after crash + cli_call index_repository "{\"repo_path\":\"$SOAK_PROJECT\"}" + echo "OK: clean re-index after crash" + else + echo "FAIL: server died during crash test" + fi +fi + +# ── Phase 6: Shutdown + analysis ───────────────────────────────── + +echo "--- Phase 6: shutdown + analysis ---" +kill "$SLEEP_PID" 2>/dev/null || true # close stdin → EOF → server exits +sleep 2 +kill "$SERVER_PID" 2>/dev/null || true +wait "$SERVER_PID" 2>/dev/null || true + +# Final diagnostics (written by thread before exit) +FINAL_DIAG="/tmp/cbm-diagnostics-${SERVER_PID}.json" + +# ── Analysis ───────────────────────────────────────────────────── + +PASS=true + +# Check 1: RSS growth +# For short runs (<10min): check absolute ceiling (warmup noise dominates slope) +# For long runs (>=10min): use linear regression on post-warmup samples (skip first 20%) +TOTAL_SAMPLES=$(awk -F, 'NR>1 && $3>0 { n++ } END { print n+0 }' "$METRICS_CSV") +MAX_RSS=$(awk -F, 'NR>1 && $3>0 { if ($3>max) max=$3 } END { printf "%.0f", max/1024/1024 }' "$METRICS_CSV") +echo "RSS max: ${MAX_RSS}MB (${TOTAL_SAMPLES} samples)" | tee -a "$SUMMARY" + +if [ "$DURATION_MIN" -lt 10 ]; then + # Short run: absolute ceiling check + if [ "${MAX_RSS:-0}" -gt 100 ] 2>/dev/null; then + echo "FAIL: RSS ${MAX_RSS}MB > 100MB ceiling" | tee -a "$SUMMARY" + PASS=false + fi +else + # Long run: slope after skipping warmup (first 20% of samples) + RSS_SLOPE=$(awk -F, -v skip="$((TOTAL_SAMPLES / 5))" ' + NR>1 && $3>0 { + row++ + if (row <= skip) next + n++; x=$1; y=$3; sx+=x; sy+=y; sxx+=x*x; sxy+=x*y + } + END { + if (n<5) { print 0; exit } + slope = (n*sxy - sx*sy) / (n*sxx - sx*sx) + printf "%.0f", slope * 3600 / 1024 + }' "$METRICS_CSV") + echo "RSS slope (post-warmup): ${RSS_SLOPE} KB/hr" | tee -a "$SUMMARY" + if [ "${RSS_SLOPE:-0}" -gt 500 ] 2>/dev/null; then + echo "FAIL: RSS slope ${RSS_SLOPE} KB/hr > 500 KB/hr threshold" | tee -a "$SUMMARY" + PASS=false + fi +fi + +# Check 2: FD drift +FD_DRIFT=$(awk -F, 'NR>1 && $5>0 { if (!first) first=$5; last=$5 } END { print last-first }' "$METRICS_CSV") +echo "FD drift: ${FD_DRIFT:-0}" | tee -a "$SUMMARY" +if [ "${FD_DRIFT:-0}" -gt 20 ] 2>/dev/null; then + echo "FAIL: FD drift ${FD_DRIFT} > 20" | tee -a "$SUMMARY" + PASS=false +fi + +# Check 3: Idle CPU +IDLE_INT=$(echo "$IDLE_CPU" | cut -d. -f1) +echo "Idle CPU: ${IDLE_CPU}%" | tee -a "$SUMMARY" +if [ "${IDLE_INT:-0}" -gt 5 ] 2>/dev/null; then + echo "FAIL: idle CPU ${IDLE_CPU}% > 5%" | tee -a "$SUMMARY" + PASS=false +fi + +# Check 4: Max query latency +MAX_LATENCY=$(awk -F, 'NR>1 { if ($3>max) max=$3 } END { print max+0 }' "$LATENCY_CSV") +echo "Max query latency: ${MAX_LATENCY}ms" | tee -a "$SUMMARY" +if [ "${MAX_LATENCY:-0}" -gt 10000 ] 2>/dev/null; then + echo "FAIL: max latency ${MAX_LATENCY}ms > 10s" | tee -a "$SUMMARY" + PASS=false +fi + +# Check 5: Query count (sanity — should have many) +TOTAL_QUERIES=$(awk -F, 'NR>1 { n++ } END { print n+0 }' "$LATENCY_CSV") +echo "Total queries: $TOTAL_QUERIES" | tee -a "$SUMMARY" + +# ── Cleanup ────────────────────────────────────────────────────── + +rm -rf "$SOAK_PROJECT" + +echo "" +if $PASS; then + echo "=== soak-test: PASSED ===" | tee -a "$SUMMARY" +else + echo "=== soak-test: FAILED ===" | tee -a "$SUMMARY" + exit 1 +fi diff --git a/src/foundation/diagnostics.c b/src/foundation/diagnostics.c new file mode 100644 index 000000000..ff6abac17 --- /dev/null +++ b/src/foundation/diagnostics.c @@ -0,0 +1,199 @@ +/* + * diagnostics.c — Periodic diagnostics file writer. + * + * Writes JSON to /tmp/cbm-diagnostics-.json every 5 seconds. + * Atomic: writes .tmp then renames to avoid partial reads. + */ +#include "foundation/diagnostics.h" +#include "foundation/mem.h" +#include "foundation/compat.h" +#include "foundation/compat_thread.h" +#include "foundation/compat_fs.h" +#include "foundation/platform.h" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#define getpid _getpid +#else +#include +#include +#endif + +/* ── Globals ─────────────────────────────────────────────────────── */ + +cbm_query_stats_t g_query_stats = {0}; +static atomic_int g_diag_stop = 0; +static cbm_thread_t g_diag_thread; +static bool g_diag_started = false; +static time_t g_start_time = 0; +static char g_diag_path[256] = ""; + +/* ── Query stats ─────────────────────────────────────────────────── */ + +void cbm_diag_record_query(long long duration_us, bool is_error) { + atomic_fetch_add(&g_query_stats.count, 1); + atomic_fetch_add(&g_query_stats.time_us, duration_us); + if (is_error) { + atomic_fetch_add(&g_query_stats.errors, 1); + } + /* Update max (lock-free CAS loop) */ + long long old_max = atomic_load(&g_query_stats.max_us); + while (duration_us > old_max) { + if (atomic_compare_exchange_weak(&g_query_stats.max_us, &old_max, duration_us)) { + break; + } + } +} + +/* ── FD count (platform-specific) ────────────────────────────────── */ + +static int count_open_fds(void) { +#ifdef __linux__ + int count = 0; + DIR *d = opendir("/proc/self/fd"); + if (d) { + while (readdir(d)) { + count++; + } + closedir(d); + count -= 2; /* . and .. */ + } + return count; +#elif defined(__APPLE__) + /* Use proc_pidinfo on macOS — but for simplicity, count via /dev/fd */ + int count = 0; + DIR *d = opendir("/dev/fd"); + if (d) { + while (readdir(d)) { + count++; + } + closedir(d); + count -= 2; /* . and .. */ + } + return count; +#else + return -1; /* Not available on Windows */ +#endif +} + +/* ── Writer ──────────────────────────────────────────────────────── */ + +#define DIAG_INTERVAL_S 5 +#define DIAG_PATH_EXTRA 24 /* ".tmp" + safety margin */ + +static void write_diagnostics(void) { + /* Collect mimalloc stats */ + size_t elapsed_ms = 0; + size_t user_ms = 0; + size_t sys_ms = 0; + size_t current_rss = 0; + size_t peak_rss = 0; + size_t current_commit = 0; + size_t peak_commit = 0; + size_t page_faults = 0; + mi_process_info(&elapsed_ms, &user_ms, &sys_ms, ¤t_rss, &peak_rss, ¤t_commit, + &peak_commit, &page_faults); + + /* Fallback RSS for ASan builds */ + if (current_rss == 0) { + current_rss = cbm_mem_rss(); + } + + int fds = count_open_fds(); + time_t now = time(NULL); + long uptime = (long)(now - g_start_time); + + int qcount = atomic_load(&g_query_stats.count); + int qerrors = atomic_load(&g_query_stats.errors); + long long qtime = atomic_load(&g_query_stats.time_us); + long long qmax = atomic_load(&g_query_stats.max_us); + long long qavg = qcount > 0 ? qtime / qcount : 0; + + /* Write to .tmp then rename (atomic) */ + char tmp_path[sizeof(g_diag_path) + DIAG_PATH_EXTRA]; + snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", g_diag_path); + + FILE *f = fopen(tmp_path, "w"); + if (!f) { + return; + } + + fprintf(f, + "{\n" + " \"uptime_s\": %ld,\n" + " \"rss_bytes\": %zu,\n" + " \"peak_rss_bytes\": %zu,\n" + " \"heap_committed_bytes\": %zu,\n" + " \"peak_committed_bytes\": %zu,\n" + " \"page_faults\": %zu,\n" + " \"fd_count\": %d,\n" + " \"query_count\": %d,\n" + " \"query_errors\": %d,\n" + " \"query_total_us\": %lld,\n" + " \"query_avg_us\": %lld,\n" + " \"query_max_us\": %lld,\n" + " \"pid\": %d\n" + "}\n", + uptime, current_rss, peak_rss, current_commit, peak_commit, page_faults, fds, qcount, + qerrors, qtime, qavg, qmax, (int)getpid()); + fclose(f); + rename(tmp_path, g_diag_path); +} + +static void *diag_thread_fn(void *arg) { + (void)arg; + while (!atomic_load(&g_diag_stop)) { + write_diagnostics(); + struct timespec ts = {DIAG_INTERVAL_S, 0}; + cbm_nanosleep(&ts, NULL); + } + /* Final write before exit */ + write_diagnostics(); + return NULL; +} + +/* ── Public API ──────────────────────────────────────────────────── */ + +bool cbm_diag_start(void) { + // NOLINTNEXTLINE(concurrency-mt-unsafe) + const char *env = getenv("CBM_DIAGNOSTICS"); + if (!env || (strcmp(env, "1") != 0 && strcmp(env, "true") != 0)) { + return false; + } + + g_start_time = time(NULL); + atomic_store(&g_diag_stop, 0); + + snprintf(g_diag_path, sizeof(g_diag_path), "%s/cbm-diagnostics-%d.json", cbm_tmpdir(), + (int)getpid()); + + if (cbm_thread_create(&g_diag_thread, 0, diag_thread_fn, NULL) != 0) { + return false; + } + + g_diag_started = true; + fprintf(stderr, "level=info msg=diagnostics.start path=%s interval=%ds\n", g_diag_path, + DIAG_INTERVAL_S); + return true; +} + +void cbm_diag_stop(void) { + if (!g_diag_started) { + return; + } + atomic_store(&g_diag_stop, 1); + cbm_thread_join(&g_diag_thread); + g_diag_started = false; + + /* Clean up file */ + cbm_unlink(g_diag_path); + char tmp_path[sizeof(g_diag_path) + DIAG_PATH_EXTRA]; + snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", g_diag_path); + cbm_unlink(tmp_path); +} diff --git a/src/foundation/diagnostics.h b/src/foundation/diagnostics.h new file mode 100644 index 000000000..b34511927 --- /dev/null +++ b/src/foundation/diagnostics.h @@ -0,0 +1,35 @@ +/* + * diagnostics.h — Periodic diagnostics file writer. + * + * When CBM_DIAGNOSTICS=1, writes /tmp/cbm-diagnostics-.json every 5s. + * Soak tests read this file to track memory, FDs, query stats over time. + */ +#ifndef CBM_DIAGNOSTICS_H +#define CBM_DIAGNOSTICS_H + +#include +#include +#include + +/* Global query stats — updated by the MCP server on each tool call. */ +typedef struct { + atomic_int count; /* total tool calls */ + atomic_int errors; /* tool calls that returned isError=true */ + atomic_llong time_us; /* cumulative wall-clock time (microseconds) */ + atomic_llong max_us; /* max single call time (microseconds) */ +} cbm_query_stats_t; + +/* Singleton query stats — MCP server increments these. */ +extern cbm_query_stats_t g_query_stats; + +/* Record a completed tool call. */ +void cbm_diag_record_query(long long duration_us, bool is_error); + +/* Start the diagnostics writer thread (if CBM_DIAGNOSTICS env is set). + * Call once from main(). Returns true if started. */ +bool cbm_diag_start(void); + +/* Stop the writer thread and delete the diagnostics file. */ +void cbm_diag_stop(void); + +#endif /* CBM_DIAGNOSTICS_H */ diff --git a/src/main.c b/src/main.c index d21c761f9..70eadcdcd 100644 --- a/src/main.c +++ b/src/main.c @@ -19,6 +19,7 @@ #include "store/store.h" #include "cli/cli.h" #include "foundation/log.h" +#include "foundation/diagnostics.h" #include "foundation/platform.h" #include "foundation/compat_thread.h" #include "foundation/mem.h" @@ -192,6 +193,7 @@ int main(int argc, char **argv) { cbm_http_server_set_binary_path(argv[0]); cbm_log_set_sink(cbm_ui_log_append); cbm_log_info("server.start", "version", CBM_VERSION); + cbm_diag_start(); /* starts if CBM_DIAGNOSTICS=1 */ /* Parse --ui and --port flags (persisted config) */ cbm_ui_config_t ui_cfg; @@ -302,6 +304,7 @@ int main(int argc, char **argv) { g_watcher = NULL; g_server = NULL; + cbm_diag_stop(); return rc; } diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 545b7c4a7..4542584b8 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -14,6 +14,7 @@ #include "cli/cli.h" #include "watcher/watcher.h" #include "foundation/mem.h" +#include "foundation/diagnostics.h" #include "foundation/platform.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" @@ -2721,7 +2722,6 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch if (strcmp(tool_name, "ingest_traces") == 0) { return handle_ingest_traces(srv, args_json); } - char msg[256]; snprintf(msg, sizeof(msg), "unknown tool: %s", tool_name); return cbm_mcp_text_result(msg, true); @@ -2997,7 +2997,17 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { char *tool_args = req.params_raw ? cbm_mcp_get_arguments(req.params_raw) : heap_strdup("{}"); + struct timespec t0; + cbm_clock_gettime(CLOCK_MONOTONIC, &t0); result_json = cbm_mcp_handle_tool(srv, tool_name, tool_args); + struct timespec t1; + cbm_clock_gettime(CLOCK_MONOTONIC, &t1); + long long dur_us = ((long long)(t1.tv_sec - t0.tv_sec) * 1000000LL) + + ((long long)(t1.tv_nsec - t0.tv_nsec) / 1000LL); + // NOLINTNEXTLINE(readability-implicit-bool-conversion) + bool is_err = (result_json != NULL) && (strstr(result_json, "\"isError\":true") != NULL); + cbm_diag_record_query(dur_us, is_err); + result_json = inject_update_notice(srv, result_json); free(tool_name); free(tool_args);