mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
Merge pull request #827 from DeusData/feat/index-supervisor-subprocess
feat(index): crash/hang supervisor — quarantine the culprit file, index the rest
This commit is contained in:
17 files changed
+1535
-11
No files matched your search
+5
-3
@@ -122,7 +122,8 @@ FOUNDATION_SRCS = \
|
||||
src/foundation/diagnostics.c \
|
||||
src/foundation/profile.c \
|
||||
src/foundation/dump_verify.c \
|
||||
src/foundation/limits.c
|
||||
src/foundation/limits.c \
|
||||
src/foundation/subprocess.c
|
||||
|
||||
# Existing extraction C code (compiled from current location)
|
||||
EXTRACTION_SRCS = \
|
||||
@@ -182,7 +183,7 @@ STORE_SRCS = src/store/store.c
|
||||
CYPHER_SRCS = src/cypher/cypher.c
|
||||
|
||||
# MCP server module (new)
|
||||
MCP_SRCS = src/mcp/mcp.c
|
||||
MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c
|
||||
|
||||
# Discover module (new)
|
||||
DISCOVER_SRCS = \
|
||||
@@ -319,7 +320,8 @@ TEST_FOUNDATION_SRCS = \
|
||||
tests/test_log.c \
|
||||
tests/test_str_util.c \
|
||||
tests/test_platform.c \
|
||||
tests/test_dump_verify.c
|
||||
tests/test_dump_verify.c \
|
||||
tests/test_subprocess.c
|
||||
|
||||
TEST_EXTRACTION_SRCS = \
|
||||
tests/test_extraction.c \
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
#include "lsp/rust_lsp.h"
|
||||
#include "preprocessor.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write
|
||||
#include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set
|
||||
#include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState
|
||||
#include "foundation/constants.h"
|
||||
#include "mimalloc.h" // mi_malloc/mi_calloc/mi_realloc/mi_free/mi_usable_size — bind 3rd-party allocators (#424)
|
||||
@@ -566,6 +568,146 @@ static int count_params_from_signature(const char *sig) {
|
||||
|
||||
// --- Main extraction function ---
|
||||
|
||||
/* Test-only deterministic fault injection for the crash/hang supervisor tests.
|
||||
* Gated entirely behind env vars that are never set in production; a matching
|
||||
* rel_path either aborts (a fault signal the supervisor classifies as a crash)
|
||||
* or spins forever (an external-scanner infinite loop the quiet-timeout kills).
|
||||
* This gives an honest guard — green iff the supervisor actually contains a real
|
||||
* fault — instead of a fixture that may stop faulting once a root cause is fixed. */
|
||||
/* Crash-supervisor per-file marker (Stage 3c skip-and-continue). In the
|
||||
* supervisor's single-threaded recovery re-run, cbm_extract_file records the
|
||||
* file it is ABOUT to process here (CBM_INDEX_MARKER_FILE) before touching it,
|
||||
* so that if this file hard-crashes the worker the parent can read the marker
|
||||
* back and learn the EXACT crasher to quarantine. Only meaningful single-
|
||||
* threaded (parallel would race the marker); the env var is set solely by the
|
||||
* supervisor during recovery, so it is a no-op on every normal run. */
|
||||
static void cbm_index_write_marker(const char *rel_path) {
|
||||
const char *mf = getenv("CBM_INDEX_MARKER_FILE");
|
||||
if (!mf || !mf[0] || !rel_path || !rel_path[0]) {
|
||||
return;
|
||||
}
|
||||
FILE *f = cbm_fopen(mf, "wb");
|
||||
if (f) {
|
||||
(void)fputs(rel_path, f);
|
||||
(void)fclose(f);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Crash-quarantine set (Stage 3c skip-and-continue) ──────────────────────
|
||||
* After a crash the supervisor re-runs the worker single-threaded, passing
|
||||
* CBM_INDEX_QUARANTINE_FILE — a newline-delimited list of repo-relative paths
|
||||
* that already crashed the indexer and MUST NOT be extracted again. Owned here,
|
||||
* next to the other env-driven extract hooks (marker + fault injector), so the
|
||||
* single hard guard lives at the one choke point every pass funnels through
|
||||
* (cbm_extract_file): whether a pass re-extracts from disk on a cache miss
|
||||
* (sequential pass_calls/usages/semantic) or extracts fresh, a quarantined file
|
||||
* short-circuits to an empty result and never reaches the parser/crash. The
|
||||
* pipeline extract loops separately REPORT the skip as phase="crash" via
|
||||
* cbm_index_is_quarantined() so the crasher surfaces in the response skipped[].
|
||||
* Loaded once, lazily; read-only after load (safe for the parallel workers,
|
||||
* though recovery runs single-threaded). Unset env ⇒ empty set ⇒ cheap no-op. */
|
||||
static CBMHashTable *g_quarantine_set = NULL;
|
||||
enum { CBM_QSET_UNINIT = 0, CBM_QSET_INITING = 1, CBM_QSET_INITED = 2 };
|
||||
static atomic_int g_quarantine_state = CBM_QSET_UNINIT;
|
||||
|
||||
static void cbm_quarantine_load(void) {
|
||||
const char *qf = getenv("CBM_INDEX_QUARANTINE_FILE");
|
||||
if (!qf || !qf[0]) {
|
||||
return; /* normal path: empty set */
|
||||
}
|
||||
FILE *f = cbm_fopen(qf, "rb");
|
||||
if (!f) {
|
||||
return;
|
||||
}
|
||||
CBMHashTable *set = cbm_ht_create(16);
|
||||
if (!set) {
|
||||
(void)fclose(f);
|
||||
return;
|
||||
}
|
||||
char line[2048];
|
||||
while (fgets(line, sizeof(line), f)) {
|
||||
size_t len = strlen(line);
|
||||
while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
|
||||
line[--len] = '\0';
|
||||
}
|
||||
if (len == 0) {
|
||||
continue;
|
||||
}
|
||||
/* Line format: "path\tphase" where phase is "crash" or "hang". A bare
|
||||
* "path" line (no tab) is tolerated and defaults to phase "crash" for
|
||||
* backward compatibility with older quarantine files. */
|
||||
char *tab = strchr(line, '\t');
|
||||
const char *phase = "crash";
|
||||
if (tab) {
|
||||
*tab = '\0';
|
||||
if (tab[1]) {
|
||||
phase = tab + 1;
|
||||
}
|
||||
}
|
||||
if (line[0] == '\0') {
|
||||
continue; /* empty path (line began with a tab) — skip */
|
||||
}
|
||||
/* The table borrows the key + value pointers, so dup both. Intentionally
|
||||
* never freed: the set lives for the whole (short-lived worker) process.
|
||||
* The value stores the phase so cbm_index_quarantine_phase() can report
|
||||
* "crash" vs "hang"; membership (cbm_index_is_quarantined) is value != NULL. */
|
||||
char *key = cbm_strdup(line);
|
||||
char *pval = cbm_strdup(phase);
|
||||
if (key && pval) {
|
||||
cbm_ht_set(set, key, (void *)pval);
|
||||
}
|
||||
}
|
||||
(void)fclose(f);
|
||||
g_quarantine_set = set;
|
||||
}
|
||||
|
||||
bool cbm_index_is_quarantined(const char *rel_path) {
|
||||
if (!rel_path || !rel_path[0]) {
|
||||
return false;
|
||||
}
|
||||
int state = atomic_load(&g_quarantine_state);
|
||||
if (state != CBM_QSET_INITED) {
|
||||
/* First caller wins the CAS and loads; racers spin until INITED.
|
||||
* Same once-init pattern as cbm_ui_log_init (http_server.c). */
|
||||
state = CBM_QSET_UNINIT;
|
||||
if (atomic_compare_exchange_strong(&g_quarantine_state, &state, CBM_QSET_INITING)) {
|
||||
cbm_quarantine_load();
|
||||
atomic_store(&g_quarantine_state, CBM_QSET_INITED);
|
||||
} else {
|
||||
while (atomic_load(&g_quarantine_state) != CBM_QSET_INITED) {
|
||||
cbm_usleep(1000); /* 1ms */
|
||||
}
|
||||
}
|
||||
}
|
||||
return g_quarantine_set && cbm_ht_has(g_quarantine_set, rel_path);
|
||||
}
|
||||
|
||||
const char *cbm_index_quarantine_phase(const char *rel_path) {
|
||||
/* cbm_index_is_quarantined drives the lazy once-load and returns true only
|
||||
* when the set is loaded and holds rel_path — so on true, g_quarantine_set is
|
||||
* non-NULL and the stored value is the phase string ("crash"/"hang"). */
|
||||
if (!cbm_index_is_quarantined(rel_path)) {
|
||||
return NULL;
|
||||
}
|
||||
return (const char *)cbm_ht_get(g_quarantine_set, rel_path);
|
||||
}
|
||||
|
||||
static void cbm_test_fault_inject(const char *rel_path) {
|
||||
if (!rel_path || !rel_path[0]) {
|
||||
return;
|
||||
}
|
||||
const char *crash_on = getenv("CBM_TEST_CRASH_ON");
|
||||
if (crash_on && crash_on[0] && strstr(rel_path, crash_on)) {
|
||||
abort(); /* SIGABRT → WIFSIGNALED → classified as a crash */
|
||||
}
|
||||
const char *hang_on = getenv("CBM_TEST_HANG_ON");
|
||||
if (hang_on && hang_on[0] && strstr(rel_path, hang_on)) {
|
||||
for (;;) {
|
||||
/* Busy-spin: the supervisor's quiet-timeout kills + reports us. */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language,
|
||||
const char *project, const char *rel_path, int64_t timeout_micros,
|
||||
const char **extra_defines, const char **include_paths) {
|
||||
@@ -579,6 +721,20 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
|
||||
cbm_arena_init(&result->arena);
|
||||
CBMArena *a = &result->arena;
|
||||
|
||||
/* Crash-quarantine hard guard (Stage 3c): a file the supervisor pinned as a
|
||||
* crasher must NEVER be parsed again. Return a clean empty result BEFORE the
|
||||
* marker write and fault injector so no pass (including sequential re-extract
|
||||
* passes that miss the result cache) can crash on it. The pipeline extract
|
||||
* loops separately record it as a phase="crash" skip. Checked before the
|
||||
* marker so quarantined files never overwrite it — the marker keeps pointing
|
||||
* at the real (non-quarantined) file being processed when a crash hits. */
|
||||
if (cbm_index_is_quarantined(rel_path)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
cbm_index_write_marker(rel_path);
|
||||
cbm_test_fault_inject(rel_path);
|
||||
|
||||
// Get language spec
|
||||
const CBMLangSpec *spec = cbm_lang_spec(language);
|
||||
if (!spec) {
|
||||
|
||||
@@ -516,6 +516,20 @@ void cbm_alloc_init(void);
|
||||
// Initialize the library. Call once at startup. Returns 0 on success.
|
||||
int cbm_init(void);
|
||||
|
||||
// True when rel_path is in the crash-quarantine set — the newline-delimited list
|
||||
// of files (CBM_INDEX_QUARANTINE_FILE) the crash supervisor pinned as crashers
|
||||
// during its single-threaded recovery re-run. Loaded once, lazily; read-only
|
||||
// after load. cbm_extract_file short-circuits such files to an empty result so no
|
||||
// pass can crash on them; the pipeline extract loops call this to also REPORT the
|
||||
// skip as phase="crash". Always false (cheap no-op) when the env var is unset.
|
||||
bool cbm_index_is_quarantined(const char *rel_path);
|
||||
|
||||
// Phase a quarantined file was pinned under: "crash" (a fault signal) or "hang"
|
||||
// (killed for making no progress). Returns NULL when rel_path is not quarantined.
|
||||
// Drives the same lazy once-load as cbm_index_is_quarantined. Used by the pipeline
|
||||
// extract loops to report the skip's phase in skipped[] (falls back to "crash").
|
||||
const char *cbm_index_quarantine_phase(const char *rel_path);
|
||||
|
||||
// Extract all data from one file. Caller must call cbm_free_result().
|
||||
// source must remain valid for the duration of the call.
|
||||
// timeout_micros: per-file parse timeout in microseconds (0 = no timeout).
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
src/foundation/compat_fs.c:popen:cbm_popen wrapper definition (POSIX)
|
||||
src/foundation/compat_fs.c:cbm_popen:cbm_popen function definition
|
||||
src/foundation/compat_fs.c:fork:cbm_exec_no_shell — fork+execvp for shell-free subprocess execution
|
||||
src/foundation/subprocess.c:fork:cbm_run_posix — fork+execv for the crash/hang-isolating index worker (supervisor primitive; child execs immediately, no code runs in the forked image)
|
||||
src/foundation/compat_fs.c:execvp:cbm_exec_no_shell — direct exec without shell interpretation
|
||||
|
||||
# ── CLI: update command (user-initiated, interactive) ──────────────────────
|
||||
|
||||
+150
-4
@@ -95,19 +95,31 @@ run_bounded() {
|
||||
fi
|
||||
local of; of="$SCRATCH/rb_out.$$"
|
||||
if [ -n "$tobin" ]; then
|
||||
"$tobin" "$secs" "$@" >"$of" 2>&1
|
||||
# -s KILL: force-kill at the deadline. The binary catches SIGTERM for a
|
||||
# graceful shutdown, so a busy-spin (e.g. a stuck external scanner) would
|
||||
# survive plain `timeout`'s TERM and hang the runner forever. timeout still
|
||||
# exits 124 on the deadline regardless of the signal used.
|
||||
"$tobin" -s KILL "$secs" "$@" >"$of" 2>&1
|
||||
RB_RC=$?
|
||||
else
|
||||
# Fallback: background the command, bound the wait via a done-fifo.
|
||||
# Fallback (no timeout/gtimeout): background the command, bound the wait via
|
||||
# a done-fifo. Open the fifo read-WRITE (exec 9<>) so the open never blocks
|
||||
# waiting for a writer — a truly-hanging command never opens the write end,
|
||||
# and a blocking open would defeat `read -t`. On the deadline, force-kill
|
||||
# the whole command subtree with SIGKILL (children first so no orphan spins
|
||||
# on): SIGTERM is caught by the binary and cannot stop a busy-spin.
|
||||
local done; done="$SCRATCH/rb_done.$$"
|
||||
rm -f "$done"; mkfifo "$done" 2>/dev/null || done=""
|
||||
( "$@" >"$of" 2>&1; echo $? > "$SCRATCH/rb_rc.$$"; [ -n "$done" ] && echo done > "$done" ) &
|
||||
local bgpid=$!
|
||||
if [ -n "$done" ]; then
|
||||
exec 9<>"$done"
|
||||
local sig=""
|
||||
read -t "$secs" sig < "$done"
|
||||
read -t "$secs" sig <&9
|
||||
exec 9<&-
|
||||
if [ -z "$sig" ]; then
|
||||
kill "$bgpid" 2>/dev/null || true
|
||||
pkill -9 -P "$bgpid" 2>/dev/null || true
|
||||
kill -9 "$bgpid" 2>/dev/null || true
|
||||
RB_RC=124 # mimic timeout's exit code
|
||||
else
|
||||
RB_RC="$(cat "$SCRATCH/rb_rc.$$" 2>/dev/null || echo 1)"
|
||||
@@ -727,6 +739,138 @@ inv_garbage_files_cli() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Invariant: a hard crash on one file is SKIPPED and the rest is indexed ──
|
||||
# Stage 3c skip-and-continue: a file that hard-crashes the native indexer
|
||||
# (SIGSEGV/abort/stack-overflow) must be QUARANTINED — the supervisor re-runs the
|
||||
# worker single-threaded, pins the exact crasher via a per-file marker, adds it to
|
||||
# a quarantine list, and re-spawns until a clean run indexes the GOOD files while
|
||||
# reporting the crasher as a phase="crash" skip. Uses the test-only fault injector
|
||||
# (CBM_TEST_CRASH_ON) so the guard is honest: with the supervisor OFF the crash
|
||||
# must genuinely escape as a signal (rc>=128, vacuity guard); with it ON (default)
|
||||
# the run must be contained (rc<128), report status="indexed" + the crasher as
|
||||
# phase="crash", index the good file (nodes>0), and NOT skip the good file.
|
||||
inv_crasher_skipped_cli() {
|
||||
local crepo="$SCRATCH/crasher_repo"
|
||||
mkdir -p "$crepo"
|
||||
printf 'def good():\n return 1\n' > "$crepo/good.py"
|
||||
printf 'def boom():\n return 2\n' > "$crepo/crash_me.py"
|
||||
git -C "$crepo" init -q 2>/dev/null || true
|
||||
local cn; cn="$(native_path "$crepo")"
|
||||
|
||||
# Honesty baseline: supervisor OFF → the injected fault must escape as a signal.
|
||||
export CBM_TEST_CRASH_ON=crash_me
|
||||
export CBM_INDEX_SUPERVISOR=0
|
||||
cli_call 60 index_repository "{\"repo_path\":\"$cn\"}"
|
||||
local base_rc="$CLI_RC"
|
||||
unset CBM_INDEX_SUPERVISOR
|
||||
|
||||
# Supervisor ON (default) → the crash must be contained AND skipped-and-continued.
|
||||
cli_call 90 index_repository "{\"repo_path\":\"$cn\"}"
|
||||
local sup_rc="$CLI_RC"
|
||||
local sup_out="$CLI_OUT"
|
||||
unset CBM_TEST_CRASH_ON
|
||||
|
||||
# Strip JSON escaping so the assertions are robust to the text-result wrapping.
|
||||
local flat
|
||||
flat="$(printf '%s' "$sup_out" | tr -d '\\' | tr -d '"')"
|
||||
local nodes
|
||||
nodes="$(printf '%s' "$sup_out" | "$PY" -c '
|
||||
import sys,re
|
||||
t=sys.stdin.read().replace("\\","").replace("\"","")
|
||||
m=re.findall(r"nodes\s*[:=]\s*(\d+)", t)
|
||||
print(max((int(x) for x in m), default=0))' 2>/dev/null)"
|
||||
|
||||
if [ "$base_rc" -le 128 ]; then
|
||||
fail "crasher-skipped-cli" "baseline did not crash (rc=$base_rc) — injector inactive, guard would be vacuous"
|
||||
elif [ "$sup_rc" -eq 124 ]; then
|
||||
fail "crasher-skipped-cli" "supervised run hung (rc=124)"
|
||||
elif [ "$sup_rc" -gt 128 ]; then
|
||||
fail "crasher-skipped-cli" "crash escaped the supervisor (signal $((sup_rc-128)))"
|
||||
elif ! printf '%s' "$flat" | grep -q 'status:indexed'; then
|
||||
fail "crasher-skipped-cli" "status not indexed after skip-and-continue: $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif ! printf '%s' "$flat" | grep -q 'phase:crash'; then
|
||||
fail "crasher-skipped-cli" "crasher not reported as phase=crash: $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif ! printf '%s' "$flat" | grep -q 'crash_me.py'; then
|
||||
fail "crasher-skipped-cli" "crash_me.py not listed as skipped: $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif printf '%s' "$flat" | grep -q 'good.py'; then
|
||||
fail "crasher-skipped-cli" "good.py was skipped (should have been indexed): $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif [ "${nodes:-0}" -le 0 ] 2>/dev/null; then
|
||||
fail "crasher-skipped-cli" "good file not indexed (nodes=${nodes:-0}): $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
else
|
||||
pass "crasher-skipped-cli (baseline rc=$base_rc escaped; supervised rc=$sup_rc: status=indexed, crash_me.py phase=crash, good.py indexed nodes=$nodes)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Invariant: a HANG on one file is SKIPPED and the rest is indexed ─────────
|
||||
# The hang twin of inv_crasher_skipped_cli. A file that makes the indexer make NO
|
||||
# progress (external-scanner infinite loop, modelled by CBM_TEST_HANG_ON's busy-
|
||||
# spin) must be QUARANTINED: the supervisor's quiet-timeout kills the worker,
|
||||
# classifies it as a HANG, pins the exact file via the marker, quarantines it as
|
||||
# phase="hang", and re-spawns until a clean run indexes the GOOD files while
|
||||
# reporting the hanger as a phase="hang" skip. Honest guard: with the supervisor
|
||||
# OFF the injected hang must genuinely NOT complete within a bound (rc=124, the
|
||||
# vacuity guard — proving the injector really hangs); with it ON + a SHORT
|
||||
# CBM_INDEX_WORKER_TIMEOUT_S the run must COMPLETE (rc<128, not 124), report
|
||||
# status="indexed" + the hanger as phase="hang", index the good file (nodes>0),
|
||||
# and NOT skip the good file.
|
||||
inv_hanger_skipped_cli() {
|
||||
local hrepo="$SCRATCH/hanger_repo"
|
||||
mkdir -p "$hrepo"
|
||||
printf 'def good():\n return 1\n' > "$hrepo/good.py"
|
||||
printf 'def slow():\n return 2\n' > "$hrepo/hang_me.py"
|
||||
git -C "$hrepo" init -q 2>/dev/null || true
|
||||
local hn; hn="$(native_path "$hrepo")"
|
||||
|
||||
# Honesty baseline: supervisor OFF → the injected hang must NOT complete within
|
||||
# the bound. The bounded runner fires (rc=124), proving the injector hangs.
|
||||
export CBM_TEST_HANG_ON=hang_me
|
||||
export CBM_INDEX_SUPERVISOR=0
|
||||
cli_call 12 index_repository "{\"repo_path\":\"$hn\"}"
|
||||
local base_rc="$CLI_RC"
|
||||
unset CBM_INDEX_SUPERVISOR
|
||||
|
||||
# Supervisor ON (default) + a SHORT no-progress timeout → the hang must be
|
||||
# detected fast, contained, and skipped-and-continued. Recovery spends two
|
||||
# ~5s timeouts (first parallel run, then the single-threaded recovery run) so
|
||||
# this invariant legitimately takes ~10-15s; it MUST still complete.
|
||||
export CBM_INDEX_WORKER_TIMEOUT_S=5
|
||||
cli_call 90 index_repository "{\"repo_path\":\"$hn\"}"
|
||||
local sup_rc="$CLI_RC"
|
||||
local sup_out="$CLI_OUT"
|
||||
unset CBM_INDEX_WORKER_TIMEOUT_S
|
||||
unset CBM_TEST_HANG_ON
|
||||
|
||||
# Strip JSON escaping so the assertions are robust to the text-result wrapping.
|
||||
local flat
|
||||
flat="$(printf '%s' "$sup_out" | tr -d '\\' | tr -d '"')"
|
||||
local nodes
|
||||
nodes="$(printf '%s' "$sup_out" | "$PY" -c '
|
||||
import sys,re
|
||||
t=sys.stdin.read().replace("\\","").replace("\"","")
|
||||
m=re.findall(r"nodes\s*[:=]\s*(\d+)", t)
|
||||
print(max((int(x) for x in m), default=0))' 2>/dev/null)"
|
||||
|
||||
if [ "$base_rc" -ne 124 ]; then
|
||||
fail "hanger-skipped-cli" "baseline did not hang (rc=$base_rc, want 124) — injector inactive, guard would be vacuous"
|
||||
elif [ "$sup_rc" -eq 124 ]; then
|
||||
fail "hanger-skipped-cli" "supervised run hung (rc=124) — hang not contained"
|
||||
elif [ "$sup_rc" -gt 128 ]; then
|
||||
fail "hanger-skipped-cli" "supervised run died via signal $((sup_rc-128))"
|
||||
elif ! printf '%s' "$flat" | grep -q 'status:indexed'; then
|
||||
fail "hanger-skipped-cli" "status not indexed after skip-and-continue: $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif ! printf '%s' "$flat" | grep -q 'phase:hang'; then
|
||||
fail "hanger-skipped-cli" "hanger not reported as phase=hang: $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif ! printf '%s' "$flat" | grep -q 'hang_me.py'; then
|
||||
fail "hanger-skipped-cli" "hang_me.py not listed as skipped: $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif printf '%s' "$flat" | grep -q 'good.py'; then
|
||||
fail "hanger-skipped-cli" "good.py was skipped (should have been indexed): $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
elif [ "${nodes:-0}" -le 0 ] 2>/dev/null; then
|
||||
fail "hanger-skipped-cli" "good file not indexed (nodes=${nodes:-0}): $(printf '%s' "$sup_out" | tr '\n' ' ' | cut -c1-300)"
|
||||
else
|
||||
pass "hanger-skipped-cli (baseline rc=$base_rc hung; supervised rc=$sup_rc: status=indexed, hang_me.py phase=hang, good.py indexed nodes=$nodes)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Invariant 8: clean exit on stdin EOF within a bounded wait (no hang) ────
|
||||
# Close the server's stdin (fd3). The server must reach EOF, break its loop, and
|
||||
# exit cleanly. We bound the wait WITHOUT sleep: closing stdin makes the server
|
||||
@@ -861,6 +1005,8 @@ inv_index_status_cli
|
||||
inv_nonexistent_repo_cli
|
||||
inv_empty_repo_cli
|
||||
inv_garbage_files_cli
|
||||
inv_crasher_skipped_cli
|
||||
inv_hanger_skipped_cli
|
||||
|
||||
# MCP server-lifecycle invariants (one shared server instance).
|
||||
inv_mcp_initialize
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
/*
|
||||
* subprocess.c — cross-platform spawn + supervise + classify.
|
||||
* See subprocess.h. The spawn/reap skeleton mirrors src/ui/http_server.c's
|
||||
* index subprocess; this generalizes it and adds crash/hang classification.
|
||||
*/
|
||||
#include "subprocess.h"
|
||||
|
||||
#include "compat.h" /* cbm_nanosleep */
|
||||
#include "platform.h" /* cbm_now_ms */
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <signal.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
/* NTSTATUS severity ERROR (top two bits set) covers the Windows crash exception
|
||||
* exit codes: 0xC0000005 (access violation), 0xC00000FD (stack overflow),
|
||||
* 0xC000001D (illegal instruction), 0xC0000094 (integer divide by zero), … */
|
||||
#define CBM_WIN_CRASH_CODE_MIN 0xC0000000u
|
||||
|
||||
#ifndef _WIN32
|
||||
static bool cbm_is_fault_signal(int sig) {
|
||||
switch (sig) {
|
||||
case SIGSEGV:
|
||||
case SIGBUS:
|
||||
case SIGILL:
|
||||
case SIGFPE:
|
||||
case SIGABRT:
|
||||
case SIGSYS:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
cbm_proc_outcome_t cbm_proc_classify(bool exited_normally, int exit_code, int term_signal,
|
||||
bool timed_out) {
|
||||
if (timed_out) {
|
||||
return CBM_PROC_HANG;
|
||||
}
|
||||
if (!exited_normally) {
|
||||
/* POSIX signal death. */
|
||||
#ifndef _WIN32
|
||||
if (cbm_is_fault_signal(term_signal)) {
|
||||
return CBM_PROC_CRASH;
|
||||
}
|
||||
#else
|
||||
(void)term_signal;
|
||||
#endif
|
||||
return CBM_PROC_KILLED;
|
||||
}
|
||||
/* Exited with a code. A Windows NTSTATUS exception code is a crash; on POSIX
|
||||
* exit codes are 0..255 so this branch never misfires there. */
|
||||
if ((unsigned)exit_code >= CBM_WIN_CRASH_CODE_MIN) {
|
||||
return CBM_PROC_CRASH;
|
||||
}
|
||||
return (exit_code == 0) ? CBM_PROC_CLEAN : CBM_PROC_EXIT_NONZERO;
|
||||
}
|
||||
|
||||
const char *cbm_proc_outcome_str(cbm_proc_outcome_t o) {
|
||||
switch (o) {
|
||||
case CBM_PROC_CLEAN:
|
||||
return "clean";
|
||||
case CBM_PROC_EXIT_NONZERO:
|
||||
return "exit_nonzero";
|
||||
case CBM_PROC_CRASH:
|
||||
return "crash";
|
||||
case CBM_PROC_HANG:
|
||||
return "hang";
|
||||
case CBM_PROC_KILLED:
|
||||
return "killed";
|
||||
case CBM_PROC_SPAWN_FAILED:
|
||||
default:
|
||||
return "spawn_failed";
|
||||
}
|
||||
}
|
||||
|
||||
/* Tail newly-appended complete lines from the child log, starting at *tail_pos.
|
||||
* A partial (non-newline-terminated) final line is left buffered: *tail_pos is
|
||||
* not advanced past it, so it is re-read once completed. Returns true if any
|
||||
* complete line was consumed (i.e. there was progress). */
|
||||
static bool cbm_tail_log(const char *log_file, long *tail_pos, cbm_proc_log_cb cb, void *ud) {
|
||||
if (!log_file) {
|
||||
return false;
|
||||
}
|
||||
FILE *lf = fopen(log_file, "r");
|
||||
if (!lf) {
|
||||
return false;
|
||||
}
|
||||
bool progressed = false;
|
||||
if (fseek(lf, *tail_pos, SEEK_SET) == 0) {
|
||||
char line[1024];
|
||||
for (;;) {
|
||||
long before = ftell(lf);
|
||||
if (!fgets(line, sizeof(line), lf)) {
|
||||
break;
|
||||
}
|
||||
size_t l = strlen(line);
|
||||
bool complete = (l > 0 && line[l - 1] == '\n');
|
||||
if (complete) {
|
||||
line[l - 1] = '\0';
|
||||
*tail_pos = ftell(lf);
|
||||
progressed = true;
|
||||
if (line[0] && cb) {
|
||||
cb(line, ud);
|
||||
}
|
||||
} else if (l == sizeof(line) - 1) {
|
||||
/* Oversized line filled the buffer without a newline — consume it
|
||||
* anyway (counts as progress) so we never stall on one long line. */
|
||||
*tail_pos = ftell(lf);
|
||||
progressed = true;
|
||||
if (cb) {
|
||||
cb(line, ud);
|
||||
}
|
||||
} else {
|
||||
/* Genuine partial final line — keep it buffered for next poll. */
|
||||
*tail_pos = before;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
fclose(lf);
|
||||
return progressed;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
static int cbm_run_win(const cbm_proc_opts_t *opts, cbm_proc_result_t *out) {
|
||||
const char *bin = opts->bin;
|
||||
const char *const default_argv[] = {bin, NULL};
|
||||
const char *const *argv = opts->argv ? opts->argv : default_argv;
|
||||
|
||||
/* Build a quoted command line from argv. */
|
||||
char cmdline[8192];
|
||||
size_t pos = 0;
|
||||
for (int i = 0; argv[i]; i++) {
|
||||
int n = snprintf(cmdline + pos, sizeof(cmdline) - pos, "%s\"%s\"", (i ? " " : ""), argv[i]);
|
||||
if (n < 0 || (size_t)n >= sizeof(cmdline) - pos) {
|
||||
out->outcome = CBM_PROC_SPAWN_FAILED;
|
||||
out->exit_code = -1;
|
||||
out->term_signal = 0;
|
||||
return -1;
|
||||
}
|
||||
pos += (size_t)n;
|
||||
}
|
||||
|
||||
HANDLE hlog = INVALID_HANDLE_VALUE;
|
||||
STARTUPINFOA si = {.cb = sizeof(si)};
|
||||
if (opts->log_file) {
|
||||
hlog = CreateFileA(opts->log_file, GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS,
|
||||
FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
if (hlog != INVALID_HANDLE_VALUE) {
|
||||
si.dwFlags = STARTF_USESTDHANDLES;
|
||||
si.hStdError = hlog;
|
||||
si.hStdOutput = hlog;
|
||||
}
|
||||
}
|
||||
|
||||
PROCESS_INFORMATION pi = {0};
|
||||
BOOL ok = CreateProcessA(NULL, cmdline, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi);
|
||||
if (hlog != INVALID_HANDLE_VALUE) {
|
||||
CloseHandle(hlog);
|
||||
}
|
||||
if (!ok) {
|
||||
out->outcome = CBM_PROC_SPAWN_FAILED;
|
||||
out->exit_code = -1;
|
||||
out->term_signal = 0;
|
||||
return -1;
|
||||
}
|
||||
|
||||
long tail_pos = 0;
|
||||
uint64_t last_activity = cbm_now_ms();
|
||||
bool timed_out = false;
|
||||
for (;;) {
|
||||
DWORD w = WaitForSingleObject(pi.hProcess, 200);
|
||||
if (cbm_tail_log(opts->log_file, &tail_pos, opts->on_log_line, opts->log_ud)) {
|
||||
last_activity = cbm_now_ms();
|
||||
}
|
||||
if (w == WAIT_OBJECT_0) {
|
||||
break;
|
||||
}
|
||||
if (opts->quiet_timeout_ms > 0 &&
|
||||
(cbm_now_ms() - last_activity) >= (uint64_t)opts->quiet_timeout_ms) {
|
||||
TerminateProcess(pi.hProcess, 1);
|
||||
WaitForSingleObject(pi.hProcess, INFINITE);
|
||||
timed_out = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
DWORD code = 1;
|
||||
GetExitCodeProcess(pi.hProcess, &code);
|
||||
CloseHandle(pi.hProcess);
|
||||
CloseHandle(pi.hThread);
|
||||
if (opts->log_file && opts->delete_log_on_exit) {
|
||||
DeleteFileA(opts->log_file);
|
||||
}
|
||||
|
||||
out->exit_code = (int)code;
|
||||
out->term_signal = 0;
|
||||
out->outcome = cbm_proc_classify(true, (int)code, 0, timed_out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#else /* POSIX */
|
||||
|
||||
static int cbm_run_posix(const cbm_proc_opts_t *opts, cbm_proc_result_t *out) {
|
||||
pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
out->outcome = CBM_PROC_SPAWN_FAILED;
|
||||
out->exit_code = -1;
|
||||
out->term_signal = 0;
|
||||
return -1;
|
||||
}
|
||||
if (pid == 0) {
|
||||
/* Child: redirect stdout+stderr to the log (or discard), then exec.
|
||||
* Use open()+dup2() (async-signal-safe, no malloc) rather than freopen():
|
||||
* the parent may be multithreaded (the MCP server holds worker/watcher/http
|
||||
* threads plus mimalloc/sqlite/libgit2 global state), and a fork() copies
|
||||
* only the calling thread — a malloc between fork and exec could deadlock on
|
||||
* a lock another thread held at fork time. open/dup2/execv touch no heap. */
|
||||
const char *bin = opts->bin;
|
||||
const char *const default_argv[] = {bin, NULL};
|
||||
const char *const *argv = opts->argv ? opts->argv : default_argv;
|
||||
const char *target = opts->log_file ? opts->log_file : "/dev/null";
|
||||
int fd = open(target, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||
if (fd >= 0) {
|
||||
(void)dup2(fd, STDOUT_FILENO);
|
||||
(void)dup2(fd, STDERR_FILENO);
|
||||
if (fd > STDERR_FILENO) {
|
||||
(void)close(fd);
|
||||
}
|
||||
}
|
||||
execv(bin, (char *const *)argv);
|
||||
_exit(127); /* exec failed */
|
||||
}
|
||||
|
||||
long tail_pos = 0;
|
||||
uint64_t last_activity = cbm_now_ms();
|
||||
bool timed_out = false;
|
||||
int wstatus = 0;
|
||||
for (;;) {
|
||||
pid_t wr;
|
||||
do {
|
||||
wr = waitpid(pid, &wstatus, WNOHANG);
|
||||
} while (wr < 0 && errno == EINTR);
|
||||
bool done = (wr == pid);
|
||||
|
||||
if (cbm_tail_log(opts->log_file, &tail_pos, opts->on_log_line, opts->log_ud)) {
|
||||
last_activity = cbm_now_ms();
|
||||
}
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
if (opts->quiet_timeout_ms > 0 &&
|
||||
(cbm_now_ms() - last_activity) >= (uint64_t)opts->quiet_timeout_ms) {
|
||||
kill(pid, SIGKILL);
|
||||
do {
|
||||
wr = waitpid(pid, &wstatus, 0);
|
||||
} while (wr < 0 && errno == EINTR);
|
||||
timed_out = true;
|
||||
break;
|
||||
}
|
||||
struct timespec ts = {0, 100000000L}; /* 100 ms poll */
|
||||
cbm_nanosleep(&ts, NULL);
|
||||
}
|
||||
|
||||
if (opts->log_file && opts->delete_log_on_exit) {
|
||||
(void)unlink(opts->log_file);
|
||||
}
|
||||
|
||||
if (WIFEXITED(wstatus)) {
|
||||
out->exit_code = WEXITSTATUS(wstatus);
|
||||
out->term_signal = 0;
|
||||
out->outcome = cbm_proc_classify(true, out->exit_code, 0, timed_out);
|
||||
} else if (WIFSIGNALED(wstatus)) {
|
||||
out->exit_code = -1;
|
||||
out->term_signal = WTERMSIG(wstatus);
|
||||
out->outcome = cbm_proc_classify(false, -1, out->term_signal, timed_out);
|
||||
} else {
|
||||
out->exit_code = -1;
|
||||
out->term_signal = 0;
|
||||
out->outcome = timed_out ? CBM_PROC_HANG : CBM_PROC_KILLED;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
int cbm_subprocess_run(const cbm_proc_opts_t *opts, cbm_proc_result_t *out) {
|
||||
cbm_proc_result_t local;
|
||||
if (!out) {
|
||||
out = &local;
|
||||
}
|
||||
out->outcome = CBM_PROC_SPAWN_FAILED;
|
||||
out->exit_code = -1;
|
||||
out->term_signal = 0;
|
||||
if (!opts || !opts->bin || !opts->bin[0]) {
|
||||
return -1;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
return cbm_run_win(opts, out);
|
||||
#else
|
||||
return cbm_run_posix(opts, out);
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* subprocess.h — spawn a child process, supervise it, and classify how it ended.
|
||||
*
|
||||
* Generalized from the crash-isolating index spawn in src/ui/http_server.c so the
|
||||
* crash/hang supervisor (Track C) can reuse one primitive across platforms.
|
||||
*
|
||||
* Beyond a plain spawn+wait it adds the two things a supervisor needs and the
|
||||
* ad-hoc harness lacked:
|
||||
* 1. Exit CLASSIFICATION — {clean, exit-nonzero, crash, hang, killed} — from
|
||||
* POSIX WIFSIGNALED/WTERMSIG and the Windows NTSTATUS exception exit codes
|
||||
* (0xC0000005 access-violation, 0xC00000FD stack-overflow, …).
|
||||
* 2. A quiet-timeout — kill + report HANG when the child makes no progress
|
||||
* (emits no new log line) for a configurable window. This catches external
|
||||
* tree-sitter scanners that infinite-loop (a hang, not a crash).
|
||||
*
|
||||
* The reap loop is EINTR-safe. Line tailing keeps a partial final line buffered
|
||||
* (an incomplete, un-newline-terminated line is not yet "progress" and is not
|
||||
* mis-read as a completed marker).
|
||||
*/
|
||||
#ifndef CBM_SUBPROCESS_H
|
||||
#define CBM_SUBPROCESS_H
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
/* How a supervised child ended. */
|
||||
typedef enum {
|
||||
CBM_PROC_CLEAN = 0, /* exited with code 0 */
|
||||
CBM_PROC_EXIT_NONZERO, /* exited with a nonzero code (a graceful failure) */
|
||||
CBM_PROC_CRASH, /* died from a fault: POSIX SIGSEGV/BUS/ILL/FPE/ABRT/SYS,
|
||||
* or a Windows NTSTATUS exception exit code (>= 0xC0000000) */
|
||||
CBM_PROC_HANG, /* made no progress within the quiet-timeout; we killed it */
|
||||
CBM_PROC_KILLED, /* terminated by a non-fault signal we did not initiate */
|
||||
CBM_PROC_SPAWN_FAILED /* fork/exec/CreateProcess failed — no child ever ran */
|
||||
} cbm_proc_outcome_t;
|
||||
|
||||
typedef struct {
|
||||
cbm_proc_outcome_t outcome;
|
||||
int exit_code; /* WEXITSTATUS / GetExitCodeProcess; -1 when terminated by a POSIX signal */
|
||||
int term_signal; /* WTERMSIG on POSIX; 0 otherwise */
|
||||
} cbm_proc_result_t;
|
||||
|
||||
/* Called for each newly-completed (newline-terminated) log line while the child
|
||||
* runs. A completed line also resets the quiet-timeout (it is progress). */
|
||||
typedef void (*cbm_proc_log_cb)(const char *line, void *ud);
|
||||
|
||||
typedef struct {
|
||||
const char *bin; /* executable path; also argv[0] when argv is NULL */
|
||||
const char *const *argv; /* NULL-terminated argv; NULL => { bin, NULL } */
|
||||
const char *log_file; /* child stdout+stderr are redirected here and tailed;
|
||||
* NULL => discard child output, no tailing */
|
||||
cbm_proc_log_cb on_log_line; /* optional per-line callback */
|
||||
void *log_ud; /* user data for on_log_line */
|
||||
int quiet_timeout_ms; /* <= 0 => no timeout; else kill+HANG after this many
|
||||
* ms with no new completed log line */
|
||||
bool delete_log_on_exit; /* unlink log_file after reaping */
|
||||
} cbm_proc_opts_t;
|
||||
|
||||
/* Spawn opts->bin, supervise (tail + optional quiet-timeout), block until it ends,
|
||||
* and classify the result into *out. Returns 0 if a child was spawned and reaped
|
||||
* (out filled), or -1 if the spawn itself failed (out->outcome == CBM_PROC_SPAWN_FAILED). */
|
||||
int cbm_subprocess_run(const cbm_proc_opts_t *opts, cbm_proc_result_t *out);
|
||||
|
||||
/* Pure outcome classifier — exposed so the platform-specific exit-code mapping
|
||||
* (notably the Windows NTSTATUS crash codes) is unit-testable on every platform.
|
||||
* exited_normally: the child returned an exit code (POSIX WIFEXITED; always true
|
||||
* on Windows, which has no signals — crashes surface as codes).
|
||||
* exit_code: the exit / exception code (meaningful when exited_normally).
|
||||
* term_signal: POSIX terminating signal (meaningful when !exited_normally).
|
||||
* timed_out: we killed the child for exceeding the quiet-timeout. */
|
||||
cbm_proc_outcome_t cbm_proc_classify(bool exited_normally, int exit_code, int term_signal,
|
||||
bool timed_out);
|
||||
|
||||
/* Stable lowercase name for an outcome (for structured logs / skip reasons). */
|
||||
const char *cbm_proc_outcome_str(cbm_proc_outcome_t o);
|
||||
|
||||
#endif /* CBM_SUBPROCESS_H */
|
||||
+39
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
#include "cbm.h" // cbm_alloc_init — bind 3rd-party allocators to mimalloc before any sqlite/git init
|
||||
#include "mcp/mcp.h"
|
||||
#include "mcp/index_supervisor.h"
|
||||
#include "watcher/watcher.h"
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "store/store.h"
|
||||
@@ -37,6 +38,7 @@ enum {
|
||||
#include "foundation/diagnostics.h"
|
||||
#include "foundation/platform.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/mem.h"
|
||||
#include "foundation/profile.h"
|
||||
@@ -231,6 +233,25 @@ static bool cli_strip_flag(int *argc, char **argv, const char *flag) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Strip a flag AND its following value from argv, returning the value (a pointer
|
||||
* into the original argv strings, valid for the process lifetime) or NULL if the
|
||||
* flag is absent. */
|
||||
static const char *cli_strip_flag_value(int *argc, char **argv, const char *flag) {
|
||||
for (int i = 0; i < *argc; i++) {
|
||||
if (strcmp(argv[i], flag) != 0) {
|
||||
continue;
|
||||
}
|
||||
const char *value = (i + SKIP_ONE < *argc) ? argv[i + SKIP_ONE] : NULL;
|
||||
int remove_count = value ? 2 : 1;
|
||||
for (int j = i; j < *argc - remove_count; j++) {
|
||||
argv[j] = argv[j + remove_count];
|
||||
}
|
||||
*argc -= remove_count;
|
||||
return value;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Portable "is fd a terminal?" — _isatty on Windows, isatty on POSIX. */
|
||||
#ifdef _WIN32
|
||||
#define cli_isatty(fd) _isatty(fd)
|
||||
@@ -299,6 +320,14 @@ static int run_cli(int argc, char **argv) {
|
||||
bool progress = cli_strip_flag(&argc, argv, "--progress");
|
||||
bool raw_json = cli_strip_flag(&argc, argv, "--json");
|
||||
|
||||
/* Supervisor worker role: when this process was spawned as a supervised index
|
||||
* worker, run indexing in-process (never re-supervise) and write the result to
|
||||
* the given file for the parent to read back. Stripped here so the tool
|
||||
* dispatch below sees only the tool name + its args. */
|
||||
bool index_worker = cli_strip_flag(&argc, argv, "--index-worker");
|
||||
const char *response_out = cli_strip_flag_value(&argc, argv, "--response-out");
|
||||
cbm_index_set_worker_role(index_worker, response_out);
|
||||
|
||||
if (argc < MAIN_MIN_ARGC) {
|
||||
(void)fprintf(stderr, CLI_USAGE);
|
||||
return SKIP_ONE;
|
||||
@@ -393,6 +422,16 @@ static int run_cli(int argc, char **argv) {
|
||||
int exit_code = 0;
|
||||
|
||||
if (result) {
|
||||
/* Supervised worker: hand the full result string to the parent via the
|
||||
* response file before printing (parent reads it back on a clean exit). */
|
||||
const char *ro = cbm_index_worker_response_out();
|
||||
if (ro) {
|
||||
FILE *rf = cbm_fopen(ro, "wb");
|
||||
if (rf) {
|
||||
(void)fputs(result, rf);
|
||||
(void)fclose(rf);
|
||||
}
|
||||
}
|
||||
if (raw_json) {
|
||||
printf("%s\n", result);
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
* index_supervisor.c — see index_supervisor.h.
|
||||
*/
|
||||
#include "index_supervisor.h"
|
||||
|
||||
#include "foundation/compat.h" /* cbm_setenv, cbm_unsetenv */
|
||||
#include "foundation/compat_fs.h" /* cbm_mkdir_p, cbm_fopen */
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/platform.h" /* cbm_resolve_cache_dir */
|
||||
#include "ui/http_server.h" /* cbm_http_server_resolve_binary_path */
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <process.h> /* _getpid */
|
||||
#define cbm_getpid _getpid
|
||||
#else
|
||||
#include <unistd.h> /* getpid */
|
||||
#define cbm_getpid getpid
|
||||
#endif
|
||||
|
||||
/* ── Worker-role state ────────────────────────────────────────────── */
|
||||
|
||||
static bool g_worker_active = false;
|
||||
static char g_worker_response_out[1024] = {0};
|
||||
|
||||
void cbm_index_set_worker_role(bool is_worker, const char *response_out) {
|
||||
g_worker_active = is_worker;
|
||||
if (response_out && response_out[0]) {
|
||||
snprintf(g_worker_response_out, sizeof(g_worker_response_out), "%s", response_out);
|
||||
} else {
|
||||
g_worker_response_out[0] = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
bool cbm_index_worker_active(void) {
|
||||
return g_worker_active;
|
||||
}
|
||||
|
||||
const char *cbm_index_worker_response_out(void) {
|
||||
return g_worker_response_out[0] ? g_worker_response_out : NULL;
|
||||
}
|
||||
|
||||
bool cbm_index_supervisor_should_wrap(void) {
|
||||
if (g_worker_active) {
|
||||
return false; /* I am the worker — run in-process, never re-supervise */
|
||||
}
|
||||
const char *sv = getenv("CBM_INDEX_SUPERVISOR");
|
||||
if (sv && strcmp(sv, "0") == 0) {
|
||||
return false; /* kill switch → in-process */
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Quiet-timeout (ms) for a supervised worker: killed + reported as a hang if it
|
||||
* emits no NEW log line within the window. This is a NO-PROGRESS timeout — every
|
||||
* completed log line the worker tails (per-batch parallel.extract.progress every
|
||||
* 10 files, plus each pass boundary) resets it — NOT a total-time cap, so a large
|
||||
* repo that keeps making progress is never falsely killed. Default: 15 min (a
|
||||
* genuinely stuck file emits nothing, so this fires only on a real hang). The
|
||||
* CBM_INDEX_WORKER_TIMEOUT_S override (seconds → ms) tightens it for tests. */
|
||||
static int worker_quiet_timeout_ms(void) {
|
||||
enum { DEFAULT_QUIET_TIMEOUT_MS = 900000 }; /* 15 min with no progress */
|
||||
const char *e = getenv("CBM_INDEX_WORKER_TIMEOUT_S");
|
||||
if (e && e[0]) {
|
||||
long s = atol(e);
|
||||
if (s > 0) {
|
||||
return (int)(s * 1000);
|
||||
}
|
||||
}
|
||||
return DEFAULT_QUIET_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
/* Read an entire file into a heap string (NUL-terminated). NULL on error. */
|
||||
static char *slurp_file(const char *path) {
|
||||
FILE *f = cbm_fopen(path, "rb");
|
||||
if (!f) {
|
||||
return NULL;
|
||||
}
|
||||
if (fseek(f, 0, SEEK_END) != 0) {
|
||||
(void)fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
long n = ftell(f);
|
||||
if (n < 0) {
|
||||
(void)fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
(void)fseek(f, 0, SEEK_SET);
|
||||
char *buf = (char *)malloc((size_t)n + 1);
|
||||
if (!buf) {
|
||||
(void)fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
size_t rd = fread(buf, 1, (size_t)n, f);
|
||||
(void)fclose(f);
|
||||
buf[rd] = '\0';
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Resolve a per-run temp path <cache_dir>/logs/.worker-<pid><suffix>. */
|
||||
static void worker_tmp_path(char *out, size_t out_sz, int pid, const char *suffix) {
|
||||
const char *cdir = cbm_resolve_cache_dir();
|
||||
if (cdir && cdir[0]) {
|
||||
char dir[900];
|
||||
snprintf(dir, sizeof(dir), "%s/logs", cdir);
|
||||
cbm_mkdir_p(dir, 0755);
|
||||
snprintf(out, out_sz, "%s/.worker-%d%s", dir, pid, suffix);
|
||||
} else {
|
||||
snprintf(out, out_sz, ".worker-%d%s", pid, suffix);
|
||||
}
|
||||
}
|
||||
|
||||
int cbm_index_spawn_worker(const char *args_json, bool single_thread, const char *marker_file,
|
||||
const char *quarantine_file, cbm_index_worker_result_t *result) {
|
||||
result->outcome = CBM_PROC_SPAWN_FAILED;
|
||||
result->exit_code = -1;
|
||||
result->term_signal = 0;
|
||||
result->response = NULL;
|
||||
|
||||
char self[1024] = {0};
|
||||
if (!cbm_http_server_resolve_binary_path(NULL, self, sizeof(self)) || !self[0]) {
|
||||
cbm_log_warn("index.supervisor.no_self_path", "action", "degrade_in_process");
|
||||
return -1;
|
||||
}
|
||||
|
||||
int pid = (int)cbm_getpid();
|
||||
char resp_path[1024];
|
||||
char log_path[1024];
|
||||
worker_tmp_path(resp_path, sizeof(resp_path), pid, ".response");
|
||||
worker_tmp_path(log_path, sizeof(log_path), pid, ".log");
|
||||
(void)remove(resp_path); /* clear any stale file */
|
||||
|
||||
/* No --progress: the worker's DEFAULT structured logging already provides the
|
||||
* no-progress heartbeat (INFO parallel.extract.progress every 10 files + each
|
||||
* pass boundary — all newline-terminated → tailed → reset the quiet-timeout).
|
||||
* --progress would be strictly worse here: it installs a REPLACE-mode sink that
|
||||
* suppresses those default lines and emits per-file extraction as a carriage-
|
||||
* return in-place update (no trailing '\n'), which cbm_tail_log does not count
|
||||
* as progress. (It would not corrupt the response either — that goes to the
|
||||
* separate --response-out file, not stdout.) */
|
||||
const char *argv[8];
|
||||
int n = 0;
|
||||
argv[n++] = self;
|
||||
argv[n++] = "cli";
|
||||
argv[n++] = "--index-worker";
|
||||
argv[n++] = "index_repository";
|
||||
argv[n++] = args_json;
|
||||
argv[n++] = "--response-out";
|
||||
argv[n++] = resp_path;
|
||||
argv[n] = NULL;
|
||||
|
||||
/* Recovery-run probe knobs → inherited env for the child. Spawns are
|
||||
* sequential, so mutating the parent's environment around a single spawn is
|
||||
* safe. Set only the requested knobs; unset them all again after reaping so
|
||||
* a later attempt (or the caller) starts from a clean environment. */
|
||||
if (single_thread) {
|
||||
cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1);
|
||||
}
|
||||
if (marker_file && marker_file[0]) {
|
||||
cbm_setenv("CBM_INDEX_MARKER_FILE", marker_file, 1);
|
||||
}
|
||||
if (quarantine_file && quarantine_file[0]) {
|
||||
cbm_setenv("CBM_INDEX_QUARANTINE_FILE", quarantine_file, 1);
|
||||
}
|
||||
|
||||
cbm_proc_opts_t opts = {0};
|
||||
opts.bin = self;
|
||||
opts.argv = argv;
|
||||
opts.log_file = log_path;
|
||||
opts.quiet_timeout_ms = worker_quiet_timeout_ms();
|
||||
opts.delete_log_on_exit = true;
|
||||
|
||||
cbm_proc_result_t r;
|
||||
int run_rc = cbm_subprocess_run(&opts, &r);
|
||||
|
||||
if (single_thread) {
|
||||
cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
|
||||
}
|
||||
if (marker_file && marker_file[0]) {
|
||||
cbm_unsetenv("CBM_INDEX_MARKER_FILE");
|
||||
}
|
||||
if (quarantine_file && quarantine_file[0]) {
|
||||
cbm_unsetenv("CBM_INDEX_QUARANTINE_FILE");
|
||||
}
|
||||
|
||||
if (run_rc != 0) {
|
||||
(void)remove(resp_path);
|
||||
cbm_log_warn("index.supervisor.spawn_failed", "action", "degrade_in_process");
|
||||
return -1;
|
||||
}
|
||||
|
||||
result->outcome = r.outcome;
|
||||
result->exit_code = r.exit_code;
|
||||
result->term_signal = r.term_signal;
|
||||
if (r.outcome == CBM_PROC_CLEAN) {
|
||||
result->response = slurp_file(resp_path);
|
||||
}
|
||||
(void)remove(resp_path);
|
||||
|
||||
char sig[16];
|
||||
snprintf(sig, sizeof(sig), "%d", r.term_signal);
|
||||
cbm_log_info("index.supervisor.reap", "outcome", cbm_proc_outcome_str(r.outcome), "signal",
|
||||
sig);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cbm_index_worker_result_free(cbm_index_worker_result_t *result) {
|
||||
if (result) {
|
||||
free(result->response);
|
||||
result->response = NULL;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* index_supervisor.h — run index_repository in a supervised worker subprocess.
|
||||
*
|
||||
* A single pathological file can hard-crash (SIGSEGV / stack overflow / abort) or
|
||||
* hang the native indexer, and today that takes down the whole MCP server or CLI.
|
||||
* The supervisor runs the actual index in a CHILD process (the same binary
|
||||
* re-invoked as `cli --index-worker index_repository …`), reaps it, and classifies
|
||||
* how it ended. A crash/hang is contained to the child; the parent survives and
|
||||
* reports it instead of dying.
|
||||
*
|
||||
* This module owns only the spawn/reap MECHANISM and the worker-role state. The
|
||||
* MCP handler (mcp.c) owns the gate placement and the response building, so this
|
||||
* module has no dependency on the response format.
|
||||
*
|
||||
* fork+exec only (never fork-and-run-in-child): the server holds persistent
|
||||
* threads plus mimalloc/sqlite/libgit2 global state with no pthread_atfork, so a
|
||||
* fork without exec would be a latent deadlock. Recursion is prevented by an argv
|
||||
* flag (`--index-worker`), never an ambient env var.
|
||||
*/
|
||||
#ifndef CBM_INDEX_SUPERVISOR_H
|
||||
#define CBM_INDEX_SUPERVISOR_H
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "foundation/subprocess.h" /* cbm_proc_outcome_t */
|
||||
|
||||
/* Worker-role state, set once from the CLI arg parser (main.c) when this process
|
||||
* was spawned as a supervised worker. When active, indexing must run in-process
|
||||
* (the gate must NOT re-supervise). response_out (may be NULL) is the file the
|
||||
* worker writes its final result string to, for the parent to read back. */
|
||||
void cbm_index_set_worker_role(bool is_worker, const char *response_out);
|
||||
bool cbm_index_worker_active(void);
|
||||
const char *cbm_index_worker_response_out(void);
|
||||
|
||||
/* True when handle_index_repository should wrap the run in a supervised child:
|
||||
* this process is not itself a worker AND the kill switch (CBM_INDEX_SUPERVISOR=0)
|
||||
* is not set. */
|
||||
bool cbm_index_supervisor_should_wrap(void);
|
||||
|
||||
typedef struct {
|
||||
cbm_proc_outcome_t outcome; /* how the worker ended */
|
||||
int exit_code; /* worker exit code (-1 if signalled) */
|
||||
int term_signal; /* POSIX terminating signal, else 0 */
|
||||
char *response; /* worker's result string on CLEAN exit (caller frees); else NULL */
|
||||
} cbm_index_worker_result_t;
|
||||
|
||||
/* Spawn `<self> cli --index-worker index_repository <args_json> --response-out <tmp>`,
|
||||
* supervise it (quiet-timeout for hangs), reap, and classify. On a clean exit,
|
||||
* result->response holds the worker's response string (read from the temp file).
|
||||
* Returns 0 if a worker was spawned and reaped (result filled), or -1 if the
|
||||
* child could not be spawned (caller degrades to in-process).
|
||||
*
|
||||
* Probe knobs for the skip-and-continue recovery re-run (Stage 3c) are passed to
|
||||
* the child as inherited env vars around the spawn (set before, unset after —
|
||||
* safe because spawns are sequential):
|
||||
* - single_thread → CBM_INDEX_SINGLE_THREAD=1: the pipeline uses exactly one
|
||||
* worker, so a per-file marker pins the EXACT crasher.
|
||||
* - marker_file → CBM_INDEX_MARKER_FILE: the worker writes the rel_path of
|
||||
* the file it is about to process here before touching it.
|
||||
* - quarantine_file → CBM_INDEX_QUARANTINE_FILE: newline-delimited rel_paths to
|
||||
* skip and report as phase="crash".
|
||||
* Any of the three may be false/NULL to leave that knob unset (the normal first
|
||||
* attempt passes single_thread=false, marker_file=NULL, quarantine_file=NULL). */
|
||||
int cbm_index_spawn_worker(const char *args_json, bool single_thread, const char *marker_file,
|
||||
const char *quarantine_file, cbm_index_worker_result_t *result);
|
||||
|
||||
void cbm_index_worker_result_free(cbm_index_worker_result_t *result);
|
||||
|
||||
#endif /* CBM_INDEX_SUPERVISOR_H */
|
||||
+247
@@ -54,6 +54,7 @@ enum {
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/log.h"
|
||||
#include "mcp/index_supervisor.h"
|
||||
#include "foundation/str_util.h"
|
||||
#include "foundation/dump_verify.h"
|
||||
#include "foundation/compat_regex.h"
|
||||
@@ -3450,7 +3451,253 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc *
|
||||
return degraded;
|
||||
}
|
||||
|
||||
/* Build the response for a worker that crashed/hung/failed without producing a
|
||||
* result. The crash is already contained (this process survived); we report it
|
||||
* rather than dying. Precise skip-and-continue (quarantine the culprit, index the
|
||||
* rest) is layered on in the probe stage. */
|
||||
static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t outcome) {
|
||||
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
|
||||
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
|
||||
yyjson_mut_val *root = yyjson_mut_obj(doc);
|
||||
yyjson_mut_doc_set_root(doc, root);
|
||||
yyjson_mut_obj_add_str(doc, root, "status", "error");
|
||||
yyjson_mut_obj_add_str(doc, root, "outcome", cbm_proc_outcome_str(outcome));
|
||||
yyjson_mut_obj_add_str(
|
||||
doc, root, "hint",
|
||||
outcome == CBM_PROC_HANG
|
||||
? "Indexing worker timed out (a file made no progress). The worker was "
|
||||
"terminated and the server survived. Re-run to retry."
|
||||
: "Indexing worker crashed on a file. The crash was contained (the server "
|
||||
"survived). Re-run to retry; a future release isolates the culprit file.");
|
||||
if (repo_path) {
|
||||
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path);
|
||||
}
|
||||
char *json = yy_doc_to_str(doc);
|
||||
yyjson_mut_doc_free(doc);
|
||||
free(repo_path);
|
||||
char *result = cbm_mcp_text_result(json, true);
|
||||
free(json);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Drop the cached store so the next query reopens whatever the worker wrote (each
|
||||
* worker is a fresh process that deletes + recreates the .db). */
|
||||
static void supervisor_invalidate_store(cbm_mcp_server_t *srv) {
|
||||
if (srv->owns_store && srv->store) {
|
||||
cbm_store_close(srv->store);
|
||||
srv->store = NULL;
|
||||
}
|
||||
free(srv->current_project);
|
||||
srv->current_project = NULL;
|
||||
}
|
||||
|
||||
/* Resolve a per-supervisor-run temp path <cache_dir>/logs/.supervisor-<pid><suffix>
|
||||
* (falls back to the CWD if the cache dir is unresolvable). Used for the crash-
|
||||
* attribution marker and the quarantine list during the recovery re-run. */
|
||||
static void supervisor_tmp_path(char *out, size_t out_sz, const char *suffix) {
|
||||
const char *cdir = cbm_resolve_cache_dir();
|
||||
if (cdir && cdir[0]) {
|
||||
char logdir[CBM_SZ_1K];
|
||||
snprintf(logdir, sizeof(logdir), "%s/logs", cdir);
|
||||
cbm_mkdir_p(logdir, 0755);
|
||||
snprintf(out, out_sz, "%s/.supervisor-%d%s", logdir, (int)getpid(), suffix);
|
||||
} else {
|
||||
snprintf(out, out_sz, ".supervisor-%d%s", (int)getpid(), suffix);
|
||||
}
|
||||
}
|
||||
|
||||
/* Read the single-line crash marker (the rel_path the worker was processing when
|
||||
* it died). Returns a trimmed heap string (caller frees), or NULL when the marker
|
||||
* is empty/unreadable (i.e. the crash could not be attributed to a file). */
|
||||
static char *supervisor_read_marker(const char *path) {
|
||||
FILE *f = cbm_fopen(path, "rb");
|
||||
if (!f) {
|
||||
return NULL;
|
||||
}
|
||||
char buf[CBM_SZ_1K];
|
||||
size_t n = fread(buf, 1, sizeof(buf) - 1, f);
|
||||
(void)fclose(f);
|
||||
buf[n] = '\0';
|
||||
size_t len = strlen(buf);
|
||||
while (len > 0 && (buf[len - 1] == '\n' || buf[len - 1] == '\r' || buf[len - 1] == ' ')) {
|
||||
buf[--len] = '\0';
|
||||
}
|
||||
return len > 0 ? cbm_strdup(buf) : NULL;
|
||||
}
|
||||
|
||||
/* Append one quarantine entry "rel\tphase\n" (phase = "crash"|"hang") to the
|
||||
* quarantine list. The worker's loader parses this back and reports the skip's
|
||||
* phase in skipped[]; a bare "rel" line is still tolerated there (defaults crash). */
|
||||
static bool supervisor_append_quarantine(const char *path, const char *rel, const char *phase) {
|
||||
FILE *f = cbm_fopen(path, "ab");
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
(void)fprintf(f, "%s\t%s\n", rel, phase);
|
||||
(void)fclose(f);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Run index_repository in a supervised worker subprocess with skip-and-continue
|
||||
* (Stage 3c). Returns the response string (caller frees):
|
||||
* - the worker's own response on a clean first run (the common path);
|
||||
* - after a crash/hang, the response from a clean single-threaded RECOVERY run
|
||||
* that quarantines the culprit file(s) — status="indexed" with them listed in
|
||||
* skipped[] as phase="crash"/"hang", and the good files indexed;
|
||||
* - a best-effort PARTIAL index (one final quarantine-only run) if the recovery
|
||||
* loop cannot converge but at least one file was quarantined;
|
||||
* - a contained-failure response only if even that cannot produce a clean run.
|
||||
* Returns NULL only when the worker could not be spawned at all, so the caller
|
||||
* degrades to the in-process path. */
|
||||
static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
|
||||
supervisor_invalidate_store(srv);
|
||||
|
||||
/* First attempt: normal parallel run. */
|
||||
cbm_index_worker_result_t wr;
|
||||
int rc = cbm_index_spawn_worker(args, false, NULL, NULL, &wr);
|
||||
|
||||
if (rc != 0 || wr.outcome == CBM_PROC_SPAWN_FAILED) {
|
||||
cbm_index_worker_result_free(&wr);
|
||||
supervisor_invalidate_store(srv);
|
||||
return NULL; /* degrade to in-process */
|
||||
}
|
||||
if (wr.outcome == CBM_PROC_CLEAN) {
|
||||
/* Clean exit → transfer the worker's response (the common path). If the
|
||||
* worker exited clean but wrote no response (a degenerate case, e.g. a
|
||||
* self binary that does not act as an index worker), resp is NULL and the
|
||||
* caller degrades to the in-process path — a clean run never needs the
|
||||
* crash-recovery loop. */
|
||||
char *resp = wr.response; /* transfer ownership to caller (may be NULL) */
|
||||
wr.response = NULL;
|
||||
cbm_index_worker_result_free(&wr);
|
||||
supervisor_invalidate_store(srv);
|
||||
return resp;
|
||||
}
|
||||
|
||||
/* Crash / hang / nonzero exit → skip-and-continue recovery. Re-run the worker
|
||||
* single-threaded with a per-file marker; each time it crashes/hangs the
|
||||
* marker names the exact crasher, which we append to the quarantine file
|
||||
* before re-spawning. A clean single-threaded run then indexes the good files
|
||||
* and reports the quarantined ones as phase="crash" skips (via the ordinary
|
||||
* Stage-2 skip plumbing — no JSON merge needed). */
|
||||
cbm_proc_outcome_t last_outcome = wr.outcome;
|
||||
cbm_index_worker_result_free(&wr);
|
||||
|
||||
char marker_path[CBM_SZ_1K];
|
||||
char quarantine_path[CBM_SZ_1K];
|
||||
supervisor_tmp_path(marker_path, sizeof(marker_path), ".marker");
|
||||
supervisor_tmp_path(quarantine_path, sizeof(quarantine_path), ".quarantine");
|
||||
(void)remove(marker_path);
|
||||
/* Start the quarantine list empty (truncate any stale file). */
|
||||
FILE *qinit = cbm_fopen(quarantine_path, "wb");
|
||||
if (qinit) {
|
||||
(void)fclose(qinit);
|
||||
}
|
||||
|
||||
int cap = 100;
|
||||
const char *cap_env = getenv("CBM_INDEX_MAX_RESTARTS");
|
||||
if (cap_env && cap_env[0]) {
|
||||
int v = atoi(cap_env);
|
||||
if (v > 0) {
|
||||
cap = v;
|
||||
}
|
||||
}
|
||||
|
||||
char *resp = NULL;
|
||||
int quarantined = 0; /* files pinned + added to the quarantine list so far */
|
||||
for (int i = 0; i < cap; i++) {
|
||||
cbm_index_worker_result_t wr2;
|
||||
int rc2 = cbm_index_spawn_worker(args, true, marker_path, quarantine_path, &wr2);
|
||||
if (rc2 != 0) {
|
||||
last_outcome = wr2.outcome;
|
||||
cbm_index_worker_result_free(&wr2);
|
||||
break; /* spawn failed mid-recovery — give up */
|
||||
}
|
||||
if (wr2.outcome == CBM_PROC_CLEAN && wr2.response) {
|
||||
resp = wr2.response; /* transfer ownership to caller */
|
||||
wr2.response = NULL;
|
||||
cbm_index_worker_result_free(&wr2);
|
||||
break; /* good files indexed; quarantined files reported as crash/hang */
|
||||
}
|
||||
if (wr2.outcome == CBM_PROC_CRASH || wr2.outcome == CBM_PROC_HANG) {
|
||||
last_outcome = wr2.outcome;
|
||||
cbm_index_worker_result_free(&wr2);
|
||||
/* crash vs hang: the phase this file is quarantined under and reported
|
||||
* as in skipped[]. A fault signal → "crash"; a no-progress kill →
|
||||
* "hang". */
|
||||
const char *phase = (last_outcome == CBM_PROC_HANG) ? "hang" : "crash";
|
||||
/* Attribute the failure to the file the marker pinned and quarantine it.
|
||||
* If the marker is empty/unreadable we cannot attribute it to a single
|
||||
* file → stop and report a contained failure. */
|
||||
char *crasher = supervisor_read_marker(marker_path);
|
||||
if (!crasher) {
|
||||
cbm_log_warn("index.supervisor.unattributable", "action", "give_up");
|
||||
break;
|
||||
}
|
||||
if (!supervisor_append_quarantine(quarantine_path, crasher, phase)) {
|
||||
cbm_log_warn("index.supervisor.quarantine_write_fail", "path", crasher);
|
||||
free(crasher);
|
||||
break;
|
||||
}
|
||||
quarantined++;
|
||||
char attempt_buf[MCP_FIELD_SIZE];
|
||||
snprintf(attempt_buf, sizeof(attempt_buf), "%d", i + 1);
|
||||
cbm_log_warn("index.file_quarantined", "path", crasher, "outcome", phase, "attempt",
|
||||
attempt_buf);
|
||||
free(crasher);
|
||||
(void)remove(marker_path); /* fresh marker for the next re-run */
|
||||
continue;
|
||||
}
|
||||
/* SPAWN_FAILED / nonzero exit / non-fault kill → not a crash we can
|
||||
* attribute; stop and report a contained failure. */
|
||||
last_outcome = wr2.outcome;
|
||||
cbm_index_worker_result_free(&wr2);
|
||||
break;
|
||||
}
|
||||
|
||||
(void)remove(marker_path); /* marker no longer needed */
|
||||
|
||||
/* Terminal best-effort-partial: the loop exited WITHOUT a clean run (cap
|
||||
* exhausted, or an unattributable failure) but at least one file was already
|
||||
* quarantined. Try ONE final single-threaded spawn with the accumulated
|
||||
* quarantine and NO marker — every known-bad file short-circuits, so a clean
|
||||
* run yields a PARTIAL index (all good files indexed, all known crashers/hangs
|
||||
* reported as skips) rather than a hard failure. Bounded by the same quiet-
|
||||
* timeout, so it cannot itself hang. Rare given monotonic progress. */
|
||||
if (!resp && quarantined > 0) {
|
||||
cbm_index_worker_result_t wrp;
|
||||
int rcp = cbm_index_spawn_worker(args, true, NULL, quarantine_path, &wrp);
|
||||
if (rcp == 0 && wrp.outcome == CBM_PROC_CLEAN && wrp.response) {
|
||||
resp = wrp.response; /* transfer ownership to caller */
|
||||
wrp.response = NULL;
|
||||
char qn[MCP_FIELD_SIZE];
|
||||
snprintf(qn, sizeof(qn), "%d", quarantined);
|
||||
cbm_log_error("index.supervisor.partial", "quarantined", qn, "outcome",
|
||||
cbm_proc_outcome_str(last_outcome));
|
||||
}
|
||||
cbm_index_worker_result_free(&wrp);
|
||||
}
|
||||
|
||||
(void)remove(quarantine_path);
|
||||
supervisor_invalidate_store(srv);
|
||||
|
||||
if (resp) {
|
||||
return resp;
|
||||
}
|
||||
return build_worker_failure_response(args, last_outcome);
|
||||
}
|
||||
|
||||
static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
/* Supervisor gate: run the index in a crash/hang-isolating worker subprocess
|
||||
* unless this process IS the worker or the kill switch (CBM_INDEX_SUPERVISOR=0)
|
||||
* is set. On spawn failure, fall through to the in-process path (degrade). */
|
||||
if (cbm_index_supervisor_should_wrap()) {
|
||||
char *supervised = index_run_supervised(srv, args);
|
||||
if (supervised) {
|
||||
return supervised;
|
||||
}
|
||||
}
|
||||
|
||||
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
|
||||
char *mode_str = cbm_mcp_get_string_arg(args, "mode");
|
||||
char *name_override = cbm_mcp_get_string_arg(args, "name");
|
||||
|
||||
@@ -514,6 +514,26 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
|
||||
const char *rel = files[i].rel_path;
|
||||
CBMLanguage lang = files[i].language;
|
||||
|
||||
/* Crash-quarantine skip (Stage 3c): the supervisor's single-threaded
|
||||
* recovery re-run always lands on THIS sequential path (worker_count
|
||||
* forced to 1). This first sequential pass REPORTS a crasher as a
|
||||
* phase="crash" skip (surfacing it in skipped[]) and continues; later
|
||||
* sequential passes (calls/usages/semantic) re-extract on a cache miss
|
||||
* but hit the hard guard inside cbm_extract_file, so they no-op without
|
||||
* re-crashing and without duplicating the skip. No-op unless
|
||||
* CBM_INDEX_QUARANTINE_FILE is set. */
|
||||
if (cbm_index_is_quarantined(rel)) {
|
||||
const char *phase = cbm_index_quarantine_phase(rel);
|
||||
if (!phase) {
|
||||
phase = "crash";
|
||||
}
|
||||
const char *reason =
|
||||
(strcmp(phase, "hang") == 0) ? "quarantined after hang" : "quarantined after crash";
|
||||
cbm_pipeline_add_file_error(ctx->pipeline, rel, reason, phase);
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Read source file */
|
||||
int source_len = 0;
|
||||
long file_size = 0;
|
||||
|
||||
@@ -666,6 +666,26 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
|
||||
const cbm_file_info_t *fi = &ec->files[file_idx];
|
||||
pp_err_list_t *errs = ec->err_lists ? &ec->err_lists[worker_id] : NULL;
|
||||
|
||||
/* Crash-quarantine skip (Stage 3c): a file the supervisor pinned as a
|
||||
* crasher must never be extracted again. Record it as a phase="crash"
|
||||
* skip in this worker's list (merged into the pipeline's file-error list
|
||||
* later, surfacing in skipped[]) and move on — the good files still
|
||||
* index and status stays "indexed". No-op unless the supervisor set
|
||||
* CBM_INDEX_QUARANTINE_FILE. Covers the parallel path; the supervisor's
|
||||
* single-threaded recovery run instead takes the sequential path
|
||||
* (pass_definitions.c), and cbm_extract_file's hard guard backstops both. */
|
||||
if (cbm_index_is_quarantined(fi->rel_path)) {
|
||||
const char *phase = cbm_index_quarantine_phase(fi->rel_path);
|
||||
if (!phase) {
|
||||
phase = "crash";
|
||||
}
|
||||
const char *reason =
|
||||
(strcmp(phase, "hang") == 0) ? "quarantined after hang" : "quarantined after crash";
|
||||
pp_err_add(errs, fi->rel_path, reason, phase);
|
||||
ws->errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Read + extract */
|
||||
int source_len = 0;
|
||||
long file_size = 0;
|
||||
|
||||
+15
-2
@@ -331,6 +331,19 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges)
|
||||
}
|
||||
}
|
||||
|
||||
/* Effective worker count. The crash supervisor re-runs its worker single-
|
||||
* threaded (CBM_INDEX_SINGLE_THREAD=1) so a per-file marker can pin the EXACT
|
||||
* crasher; a parallel re-run would race the marker. Honour that override
|
||||
* everywhere the worker count drives the parallel/sequential decision, so the
|
||||
* whole extraction phase collapses to the deterministic sequential path. */
|
||||
static int effective_worker_count(bool initial) {
|
||||
const char *st = getenv("CBM_INDEX_SINGLE_THREAD");
|
||||
if (st && st[0] == '1') {
|
||||
return 1;
|
||||
}
|
||||
return cbm_default_worker_count(initial);
|
||||
}
|
||||
|
||||
/* Resolve the DB path for this pipeline. Caller must free(). */
|
||||
static char *resolve_db_path(const cbm_pipeline_t *p) {
|
||||
char *path = malloc(CBM_SZ_1K);
|
||||
@@ -1098,7 +1111,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
|
||||
gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result};
|
||||
|
||||
if (p->mode != CBM_MODE_FAST) {
|
||||
if (cbm_default_worker_count(true) > SKIP_ONE) {
|
||||
if (effective_worker_count(true) > SKIP_ONE) {
|
||||
if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) {
|
||||
gh_threaded = true;
|
||||
}
|
||||
@@ -1187,7 +1200,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
int worker_count = cbm_default_worker_count(true);
|
||||
int worker_count = effective_worker_count(true);
|
||||
CBM_PROF_START(t_extract_total);
|
||||
int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL)
|
||||
? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t)
|
||||
|
||||
@@ -786,7 +786,11 @@ static void *index_thread_fn(void *arg) {
|
||||
|
||||
/* Build command line for CreateProcess */
|
||||
char cmdline[2048];
|
||||
snprintf(cmdline, sizeof(cmdline), "\"%s\" cli index_repository \"%s\"", bin, json_arg);
|
||||
/* --index-worker: this http_server spawn is already the crash-isolation layer,
|
||||
* so the child runs indexing in-process rather than spawning its own supervisor
|
||||
* (avoids redundant process nesting). */
|
||||
snprintf(cmdline, sizeof(cmdline), "\"%s\" cli --index-worker index_repository \"%s\"", bin,
|
||||
json_arg);
|
||||
|
||||
cbm_log_info("ui.index.spawn", "bin", bin, "log", log_file);
|
||||
|
||||
@@ -854,7 +858,7 @@ static void *index_thread_fn(void *arg) {
|
||||
FILE *lf = freopen(log_file, "w", stderr);
|
||||
(void)lf;
|
||||
freopen("/dev/null", "w", stdout);
|
||||
execl(bin, bin, "cli", "index_repository", json_arg, (char *)NULL);
|
||||
execl(bin, bin, "cli", "--index-worker", "index_repository", json_arg, (char *)NULL);
|
||||
_exit(127);
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ extern void suite_str_intern(void);
|
||||
extern void suite_log(void);
|
||||
extern void suite_str_util(void);
|
||||
extern void suite_platform(void);
|
||||
extern void suite_subprocess(void);
|
||||
extern void suite_extraction(void);
|
||||
extern void suite_extraction_inheritance(void);
|
||||
extern void suite_extraction_imports(void);
|
||||
@@ -145,6 +146,7 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(log);
|
||||
RUN_SELECTED_SUITE(str_util);
|
||||
RUN_SELECTED_SUITE(platform);
|
||||
RUN_SELECTED_SUITE(subprocess);
|
||||
RUN_SELECTED_SUITE(dump_verify);
|
||||
|
||||
/* Existing C code regression tests */
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
* test_subprocess.c — foundation/subprocess: spawn + supervise + classify.
|
||||
*
|
||||
* Two layers:
|
||||
* 1. cbm_proc_classify() — pure, exercised on EVERY platform (so the Windows
|
||||
* NTSTATUS crash-code mapping is guarded on Linux/macOS CI too, not just an
|
||||
* untested Windows branch).
|
||||
* 2. cbm_subprocess_run() — real spawn/reap, exercised on POSIX via /bin/sh
|
||||
* (SKIP_PLATFORM on Windows, which lacks it).
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include "../src/foundation/subprocess.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <signal.h>
|
||||
#endif
|
||||
|
||||
/* ── Layer 1: pure classifier (all platforms) ─────────────────────────────── */
|
||||
|
||||
TEST(subprocess_classify_clean) {
|
||||
ASSERT_EQ(cbm_proc_classify(true, 0, 0, false), CBM_PROC_CLEAN);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(subprocess_classify_exit_nonzero) {
|
||||
ASSERT_EQ(cbm_proc_classify(true, 3, 0, false), CBM_PROC_EXIT_NONZERO);
|
||||
ASSERT_EQ(cbm_proc_classify(true, 127, 0, false), CBM_PROC_EXIT_NONZERO);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Windows exception exit codes classify as CRASH — the key reason to unit-test
|
||||
* the classifier cross-platform. 0xC0000005 = access violation (SIGSEGV analog),
|
||||
* 0xC00000FD = stack overflow (the #668 reporter's 0xC00000FD). */
|
||||
TEST(subprocess_classify_windows_crash_codes) {
|
||||
ASSERT_EQ(cbm_proc_classify(true, (int)0xC0000005u, 0, false), CBM_PROC_CRASH);
|
||||
ASSERT_EQ(cbm_proc_classify(true, (int)0xC00000FDu, 0, false), CBM_PROC_CRASH);
|
||||
ASSERT_EQ(cbm_proc_classify(true, (int)0xC000001Du, 0, false), CBM_PROC_CRASH);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(subprocess_classify_posix_fault_signal_is_crash) {
|
||||
#ifndef _WIN32
|
||||
ASSERT_EQ(cbm_proc_classify(false, -1, SIGSEGV, false), CBM_PROC_CRASH);
|
||||
ASSERT_EQ(cbm_proc_classify(false, -1, SIGABRT, false), CBM_PROC_CRASH);
|
||||
ASSERT_EQ(cbm_proc_classify(false, -1, SIGBUS, false), CBM_PROC_CRASH);
|
||||
#endif
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(subprocess_classify_non_fault_signal_is_killed) {
|
||||
#ifndef _WIN32
|
||||
ASSERT_EQ(cbm_proc_classify(false, -1, SIGTERM, false), CBM_PROC_KILLED);
|
||||
ASSERT_EQ(cbm_proc_classify(false, -1, SIGKILL, false), CBM_PROC_KILLED);
|
||||
#endif
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* timed_out dominates every other signal — a killed-for-hang child is HANG,
|
||||
* not KILLED, even though we deliver SIGKILL/TerminateProcess to end it. */
|
||||
TEST(subprocess_classify_timeout_dominates) {
|
||||
ASSERT_EQ(cbm_proc_classify(false, -1, 9 /*SIGKILL*/, true), CBM_PROC_HANG);
|
||||
ASSERT_EQ(cbm_proc_classify(true, 0, 0, true), CBM_PROC_HANG);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(subprocess_outcome_str) {
|
||||
ASSERT_STR_EQ(cbm_proc_outcome_str(CBM_PROC_CLEAN), "clean");
|
||||
ASSERT_STR_EQ(cbm_proc_outcome_str(CBM_PROC_CRASH), "crash");
|
||||
ASSERT_STR_EQ(cbm_proc_outcome_str(CBM_PROC_HANG), "hang");
|
||||
ASSERT_STR_EQ(cbm_proc_outcome_str(CBM_PROC_EXIT_NONZERO), "exit_nonzero");
|
||||
ASSERT_STR_EQ(cbm_proc_outcome_str(CBM_PROC_KILLED), "killed");
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */
|
||||
|
||||
#ifndef _WIN32
|
||||
static cbm_proc_result_t run_sh(const char *script, int quiet_timeout_ms) {
|
||||
const char *argv[] = {"/bin/sh", "-c", script, NULL};
|
||||
cbm_proc_opts_t opts = {0};
|
||||
opts.bin = "/bin/sh";
|
||||
opts.argv = argv;
|
||||
opts.log_file = NULL;
|
||||
opts.quiet_timeout_ms = quiet_timeout_ms;
|
||||
cbm_proc_result_t r;
|
||||
cbm_subprocess_run(&opts, &r);
|
||||
return r;
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(subprocess_run_clean) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("POSIX /bin/sh spawn");
|
||||
#else
|
||||
cbm_proc_result_t r = run_sh("exit 0", 0);
|
||||
ASSERT_EQ(r.outcome, CBM_PROC_CLEAN);
|
||||
ASSERT_EQ(r.exit_code, 0);
|
||||
PASS();
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(subprocess_run_exit_nonzero) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("POSIX /bin/sh spawn");
|
||||
#else
|
||||
cbm_proc_result_t r = run_sh("exit 7", 0);
|
||||
ASSERT_EQ(r.outcome, CBM_PROC_EXIT_NONZERO);
|
||||
ASSERT_EQ(r.exit_code, 7);
|
||||
PASS();
|
||||
#endif
|
||||
}
|
||||
|
||||
/* A child that dies of SIGSEGV must classify as CRASH — NOT exit_nonzero and NOT
|
||||
* killed. This is the whole point of the primitive: distinguish a crash from a
|
||||
* clean failure so the supervisor can quarantine the culprit. */
|
||||
TEST(subprocess_run_crash_is_crash) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("POSIX signal semantics");
|
||||
#else
|
||||
cbm_proc_result_t r = run_sh("kill -SEGV $$", 0);
|
||||
ASSERT_EQ(r.outcome, CBM_PROC_CRASH);
|
||||
ASSERT_EQ(r.term_signal, SIGSEGV);
|
||||
PASS();
|
||||
#endif
|
||||
}
|
||||
|
||||
/* A child that makes no progress within the quiet-timeout is killed and reported
|
||||
* as HANG — the sibling failure mode of a crash (external-scanner infinite loop). */
|
||||
TEST(subprocess_run_hang_is_hang) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("POSIX /bin/sh spawn");
|
||||
#else
|
||||
cbm_proc_result_t r = run_sh("sleep 30", 300 /* ms quiet-timeout */);
|
||||
ASSERT_EQ(r.outcome, CBM_PROC_HANG);
|
||||
PASS();
|
||||
#endif
|
||||
}
|
||||
|
||||
/* A spawn of a non-existent binary fails cleanly (no child), not a crash. */
|
||||
TEST(subprocess_run_spawn_failure) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("POSIX exec semantics");
|
||||
#else
|
||||
/* execv of a bogus path: fork succeeds, child _exit(127). We classify the
|
||||
* reaped 127 as exit_nonzero — spawn_failed is reserved for fork() failing. */
|
||||
const char *argv[] = {"/nonexistent/cbm-bogus-binary", NULL};
|
||||
cbm_proc_opts_t opts = {0};
|
||||
opts.bin = "/nonexistent/cbm-bogus-binary";
|
||||
opts.argv = argv;
|
||||
cbm_proc_result_t r;
|
||||
int rc = cbm_subprocess_run(&opts, &r);
|
||||
ASSERT_EQ(rc, 0); /* fork itself succeeded */
|
||||
ASSERT_EQ(r.outcome, CBM_PROC_EXIT_NONZERO);
|
||||
ASSERT_EQ(r.exit_code, 127);
|
||||
PASS();
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(subprocess_run_null_bin_rejected) {
|
||||
cbm_proc_opts_t opts = {0};
|
||||
opts.bin = NULL;
|
||||
cbm_proc_result_t r;
|
||||
int rc = cbm_subprocess_run(&opts, &r);
|
||||
ASSERT_EQ(rc, -1);
|
||||
ASSERT_EQ(r.outcome, CBM_PROC_SPAWN_FAILED);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(subprocess) {
|
||||
RUN_TEST(subprocess_classify_clean);
|
||||
RUN_TEST(subprocess_classify_exit_nonzero);
|
||||
RUN_TEST(subprocess_classify_windows_crash_codes);
|
||||
RUN_TEST(subprocess_classify_posix_fault_signal_is_crash);
|
||||
RUN_TEST(subprocess_classify_non_fault_signal_is_killed);
|
||||
RUN_TEST(subprocess_classify_timeout_dominates);
|
||||
RUN_TEST(subprocess_outcome_str);
|
||||
RUN_TEST(subprocess_run_clean);
|
||||
RUN_TEST(subprocess_run_exit_nonzero);
|
||||
RUN_TEST(subprocess_run_crash_is_crash);
|
||||
RUN_TEST(subprocess_run_hang_is_hang);
|
||||
RUN_TEST(subprocess_run_spawn_failure);
|
||||
RUN_TEST(subprocess_run_null_bin_rejected);
|
||||
}
|
||||
Reference in new issue
Block a user