nprasad2077/codebase-memory-mcp
1
0 Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-04 10:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(daemon): fail the client fast when the daemon cannot start

Browse Source 
A daemon that died at listener publication (a full /tmp) took the
lifetime reservation, failed, and released it; the waiting client saw the
generation vanish, respawned it, and repeated that for its whole 30 s
deadline before reporting "CBM daemon is active or starting but could not
accept this client" -- the opposite of what happened, with no cause and
no path. MCP hosts then dropped the server silently (#1828).

A detached daemon has no stderr and no channel back to its launcher, so
the host now appends one start-failure record per failed start to
<cache>/logs/cbm-daemon.start-failures.log -- next to cbm-daemon.log,
through the same owner-only private-log open. Deliberately NOT the
runtime directory: that is exactly what a full /tmp cannot write, and a
new artifact there would entangle the stale-generation cleanup that
#1894 just hardened. The record carries the listener detail from the
previous commit (component, stage, errno, path) plus the runtime
directory and a wall-clock stamp, and is written while the lifetime
reservation is still held, so a client that watches the generation
vanish already finds the cause.

The client bootstrap loop reads that record (every 50 ms after its own
spawn, and unconditionally before any relaunch), accepts only records for
its endpoint not older than its spawn instant (2 s skew), and stops at
once:

    error: CBM daemon failed to start: pending_publication failed with
    ENOSPC (No space left on device) at /tmp/cbm-daemon-1000/cbm-<key>.sock.pending.tmp;
    see ~/.cache/codebase-memory-mcp/logs/cbm-daemon.log

The read is an optional bootstrap op (unit contracts with fake ops are
untouched); the production op resolves the cache log directory. Every
other wait state -- a live or stopping generation, a mute holder, a
concurrent starter -- keeps its existing semantics and diagnostics.

Tests (daemon_bootstrap):
- daemon_bootstrap_fails_fast_when_daemon_dies_at_publication: the
  production bootstrap (real cohort, probe, startup lock, handoff) with a
  seam-only spawn override that forks a REAL daemon host whose record
  writes fail with ENOSPC. RED before: 34.7 s and "active or starting";
  GREEN after: FAILED in well under 5 s naming ENOSPC and the temp path,
  one child spawned, daemon log carries errno=ENOSPC path=... .
- daemon_bootstrap_start_failure_record_round_trip: record/read/format
  contract, a stale record and another endpoint's record are ignored.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin VogelandClaude Fable 5.1 committed 2026-09-08 02:01:33 +02:00
1 parent 33c4fd80ef
commit 00060cec85
4 files changed
+673 -3
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/daemon/bootstrap.c
+338 -2
View File
@@ -6,13 +6,17 @@
#include "daemon/ipc.h"
#include "daemon/service.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/log.h"
#include "foundation/platform.h"
#include <errno.h>
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
@@ -38,6 +42,14 @@ extern char **environ;
enum {
BOOTSTRAP_RETRY_NS = 1000000,
/* #1828: how often a waiting client re-reads the start-failure log
* (the wait loop itself ticks every millisecond). */
BOOTSTRAP_START_FAILURE_CHECK_MS = 50,
/* A record this many seconds OLDER than the client's own spawn instant is
* still accepted: a sibling client's attempt that just failed is the same
* evidence, and second-granularity clocks need the slack anyway. */
BOOTSTRAP_START_FAILURE_SKEW_S = 2,
BOOTSTRAP_START_FAILURE_LOG_CAP = 65536,
BOOTSTRAP_COORDINATION_CLEANUP_MS = 500,
BOOTSTRAP_PATH_CAP = 4096,
};
@@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) {
#endif
}
static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log";
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) {
if (!out || capacity == 0) {
return false;
}
out[0] = '\0';
const char *cache = cbm_resolve_cache_dir();
if (!cache || !cache[0]) {
return false;
}
int written = snprintf(out, capacity, "%s/logs", cache);
if (written <= 0 || (size_t)written >= capacity) {
out[0] = '\0';
return false;
}
return true;
}
/* One record field: the line format is tab-separated, so a tab or newline in
* a value (a path, in theory) becomes '?' rather than a parser ambiguity. */
static void bootstrap_record_field(char *out, size_t capacity, const char *value) {
size_t used = 0;
for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) {
unsigned char ch = (unsigned char)*cursor;
out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch;
}
out[used] = '\0';
}
static uint64_t bootstrap_current_pid(void) {
#ifdef _WIN32
return (uint64_t)GetCurrentProcessId();
#else
return (uint64_t)getpid();
#endif
}
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component) {
if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) {
return false;
}
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
if (!runtime_dir || !runtime_dir[0]) {
return false;
}
cbm_daemon_ipc_listen_failure_t detail;
if (!cbm_daemon_ipc_listen_failure_detail(&detail)) {
memset(&detail, 0, sizeof(detail));
}
char safe_runtime_dir[BOOTSTRAP_PATH_CAP];
char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP];
char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir);
bootstrap_record_field(safe_component, sizeof(safe_component), component);
bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage);
bootstrap_record_field(safe_path, sizeof(safe_path), detail.path);
FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME,
BOOTSTRAP_START_FAILURE_LOG_CAP);
if (!stream) {
return false;
}
int written =
fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL),
(unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component,
safe_stage, detail.errno_value, safe_path);
bool ok = written > 0 && fflush(stream) == 0;
if (fclose(stream) != 0) {
ok = false;
}
return ok;
}
/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line
* (line is NUL-terminated, newline already stripped, and is modified). */
static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out,
cbm_daemon_bootstrap_start_failure_t *out) {
char *fields[8];
size_t count = 0;
char *cursor = line;
while (count < 8) {
fields[count++] = cursor;
char *tab = strchr(cursor, '\t');
if (!tab) {
break;
}
*tab = '\0';
cursor = tab + 1;
}
if (count != 8 || strcmp(fields[0], "v1") != 0) {
return false;
}
memset(out, 0, sizeof(*out));
out->recorded_at_s = strtoull(fields[1], NULL, 10);
out->pid = strtoull(fields[2], NULL, 10);
*runtime_dir_out = fields[3];
(void)snprintf(out->component, sizeof(out->component), "%s", fields[4]);
(void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]);
out->errno_value = (int)strtol(fields[6], NULL, 10);
(void)snprintf(out->path, sizeof(out->path), "%s", fields[7]);
return true;
}
static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) {
char path[BOOTSTRAP_PATH_CAP];
int written =
snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME);
if (written <= 0 || written >= (int)sizeof(path)) {
*status_out = -1;
return NULL;
}
#ifndef _WIN32
struct stat status;
if (lstat(path, &status) != 0) {
*status_out = errno == ENOENT ? 0 : -1;
return NULL;
}
if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) {
*status_out = -1;
return NULL;
}
#endif
/* The directory is the trust boundary (owner-only, no symlinked
* ancestry); the same validation guards cbm-daemon.log itself. */
if (!cbm_daemon_ipc_private_directory_secure(log_directory)) {
*status_out = -1;
return NULL;
}
FILE *file = cbm_fopen(path, "rb");
if (!file) {
*status_out = errno == ENOENT ? 0 : -1;
return NULL;
}
*status_out = 1;
return file;
}
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure) {
if (!out_failure) {
return -1;
}
memset(out_failure, 0, sizeof(*out_failure));
if (!log_directory || !log_directory[0] || !endpoint) {
return -1;
}
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
if (!runtime_dir || !runtime_dir[0]) {
return -1;
}
int status = 0;
FILE *file = bootstrap_start_failure_open(log_directory, &status);
if (!file) {
return status;
}
char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U);
if (!buffer) {
(void)fclose(file);
return -1;
}
size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file);
bool read_ok = !ferror(file);
(void)fclose(file);
buffer[used] = '\0';
int found = 0;
if (read_ok) {
char *line = buffer;
while (line && *line) {
char *end = strchr(line, '\n');
if (!end) {
break; /* a partial trailing line is a record still being written */
}
*end = '\0';
char *record_runtime_dir = NULL;
cbm_daemon_bootstrap_start_failure_t candidate;
if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) &&
strcmp(record_runtime_dir, runtime_dir) == 0 &&
candidate.recorded_at_s >= not_before_s) {
*out_failure = candidate; /* later lines are newer */
found = 1;
}
line = end + 1;
}
} else {
found = -1;
}
free(buffer);
return found;
}
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
const char *log_directory, char *out,
size_t capacity) {
if (!out || capacity == 0) {
return;
}
if (!failure) {
out[0] = '\0';
return;
}
const char *what = failure->stage[0] ? failure->stage
: failure->component[0] ? failure->component
: "startup";
char where[BOOTSTRAP_PATH_CAP];
if (log_directory && log_directory[0]) {
(void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory);
} else {
(void)snprintf(where, sizeof(where), "see the daemon log");
}
if (failure->errno_value != 0 && failure->path[0]) {
(void)snprintf(out, capacity,
"CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what,
cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
failure->path, where);
} else if (failure->errno_value != 0) {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s",
what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
where);
} else if (failure->path[0]) {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what,
failure->path, where);
} else {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where);
}
}
/* After this attempt spawned a daemon: did that daemon (or a sibling attempt
* moments earlier) record a start failure? A found record ends the wait with
* its cause -- respawning a deterministically failing daemon until the
* deadline only produced "active or starting" (#1828). */
static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config,
const cbm_daemon_bootstrap_ops_t *ops,
uint64_t spawn_wall_s, bool force,
uint64_t *next_check_ms,
cbm_daemon_bootstrap_result_t *result_out) {
if (spawn_wall_s == 0 || !ops->start_failure_probe) {
return false;
}
uint64_t now_ms = cbm_now_ms();
if (!force && now_ms < *next_check_ms) {
return false;
}
*next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS;
uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S
? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S
: 0;
cbm_daemon_bootstrap_start_failure_t failure;
if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) {
return false;
}
char logs[BOOTSTRAP_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
logs[0] = '\0';
}
cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message,
sizeof(result_out->message));
return true;
}
static void bootstrap_pause(uint64_t deadline) {
uint64_t now = cbm_now_ms();
if (now >= deadline) {
@@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
bool lock_acquired = false;
bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL;
uint64_t spawn_wall_s = 0;
uint64_t next_failure_check_ms = 0;
bool start_failed = false;
while (cbm_now_ms() < deadline) {
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
if (!bootstrap_probe_is_waitable(probe)) {
break;
}
@@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
break;
}
/* Never relaunch a daemon whose predecessor already recorded why it
* could not start; the record is written before that daemon lets go
* of its lifetime reservation, so this unconditional read sees it. */
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
cbm_daemon_bootstrap_launch_spec_t spec;
bool spec_ready =
config->spawn_permanent
? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec)
: cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec);
spawn_wall_s = (uint64_t)time(NULL);
if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) ||
!ops->spawn_daemon(ops->context, &spec)) {
probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR;
@@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
if (!bootstrap_probe_is_waitable(probe)) {
break;
}
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
} while (cbm_now_ms() < deadline);
if (start_failed) {
break;
}
if (!lock_acquired) {
continue;
}
@@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
if (muted_holder_pid != 0) {
if (start_failed) {
/* result_out->message already names the recorded cause. */
} else if (muted_holder_pid != 0) {
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
* hand from process, pipe, and log correlation: name the pid that
* holds the endpoint without answering, and say what to do with it. */
@@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock
return released;
}
static int bootstrap_production_start_failure_probe(
void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure) {
(void)context;
char logs[BOOTSTRAP_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
return -1;
}
return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure);
}
static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) {
(void)context;
return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock);
@@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message)
(void)fflush(stderr);
}
#ifdef CBM_ENABLE_TEST_SEAMS
static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test;
static void *g_bootstrap_spawn_override_context_for_test;
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
void *context) {
g_bootstrap_spawn_override_context_for_test = context;
g_bootstrap_spawn_override_for_test = spawn;
}
static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) {
(void)context;
return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec);
}
#endif
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) {
bootstrap_production_context_t context = {0};
const cbm_daemon_bootstrap_ops_t ops = {
cbm_daemon_bootstrap_ops_t ops = {
.context = &context,
.cohort_acquire = bootstrap_production_cohort_acquire,
.cohort_release = bootstrap_production_cohort_release,
@@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
.startup_lock_release = bootstrap_production_unlock,
.spawn_daemon = bootstrap_production_spawn,
.visible_diagnostic = bootstrap_production_diagnostic,
.start_failure_probe = bootstrap_production_start_failure_probe,
};
#ifdef CBM_ENABLE_TEST_SEAMS
if (g_bootstrap_spawn_override_for_test) {
ops.spawn_daemon = bootstrap_seam_spawn;
}
#endif
return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out);
}
src/daemon/bootstrap.h
+58
View File
@@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
typedef void *cbm_daemon_bootstrap_lock_t;
typedef void *cbm_daemon_bootstrap_cohort_t;
/* Durable daemon start-failure record (#1828). A detached daemon has no
* stderr and no channel back to the client that launched it; when its
* listener publication fails (a full /tmp, for instance) the client used to
* respawn it for the whole startup deadline and then report "active or
* starting". The host now appends one record per failed start to
* <cache>/logs/cbm-daemon.start-failures.log -- the same owner-only log
* directory as cbm-daemon.log, deliberately NOT the runtime directory: that
* one is exactly what a full /tmp cannot write, and a new artifact there
* would entangle stale-generation cleanup. A waiting client reads the newest
* record for its endpoint that is not older than its own spawn and fails
* immediately with the cause. Diagnostic only: policy never reads it. */
enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 };
typedef struct {
uint64_t recorded_at_s; /* wall clock, seconds */
uint64_t pid;
char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */
int errno_value; /* 0 when unknown */
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */
} cbm_daemon_bootstrap_start_failure_t;
/* <cache>/logs, the owner-only directory that holds every daemon log. */
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity);
/* Append one record for this process's most recent listener failure detail
* (cbm_daemon_ipc_listen_failure_detail) under `component`. */
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component);
/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1
* and fills out_failure, 0 when none matches, -1 when the log directory or
* file cannot be validated. */
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure);
/* The client-facing sentence: "CBM daemon failed to start: <stage> failed
* with ENOSPC (No space left on device) at <path>; see <log>". */
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
const char *log_directory, char *out,
size_t capacity);
/* Injectable OS/runtime boundary used by the deterministic unit contract.
* Production callers use cbm_daemon_bootstrap_execute(), whose built-in
* operations delegate to daemon IPC/runtime and write visible diagnostics to
@@ -149,6 +190,12 @@ typedef struct {
bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io);
bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec);
void (*visible_diagnostic)(void *context, const char *message);
/* Optional. After this attempt spawned a daemon, report a start-failure
* record for the endpoint not older than not_before_s (1 found, 0 none,
* -1 unreadable). A found record ends the wait immediately (#1828). */
int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure);
} cbm_daemon_bootstrap_ops_t;
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
@@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
/* Test seam for the same state machine. All callbacks are synchronous and
* borrowed for the duration of the call. */
#ifdef CBM_ENABLE_TEST_SEAMS
/* Replace ONLY the production spawn while every other production operation
* (cohort, probe, startup lock, handoff) stays real. A test forks a real
* daemon host in place of exec'ing the product binary, so the complete
* client/daemon rendezvous runs in one deterministic process tree (#1828). */
typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context,
const cbm_daemon_bootstrap_launch_spec_t *spec);
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
void *context);
#endif
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
cbm_daemon_bootstrap_result_t *result_out);
src/daemon/host.c
+20 -1
View File
@@ -5,6 +5,7 @@
#include "daemon/host_internal.h"
#include "daemon/application.h"
#include "daemon/bootstrap.h"
#include "daemon/runtime.h"
#include "daemon/project_lock.h"
#include "daemon/version_cohort.h"
@@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) {
return true;
}
/* #1828: a detached daemon has no stderr; its start failure must reach the
* client that is waiting for it, or that client burns its whole deadline and
* reports the opposite of the truth. */
static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component) {
char logs[HOST_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) ||
!cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) {
cbm_log_error("daemon.start_failure_record_failed", "component", component);
}
}
static void host_log_close(void) {
cbm_log_set_sink(NULL);
if (g_host_log_mutex_initialized) {
@@ -1051,10 +1064,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) {
};
cbm_daemon_runtime_service_t *service =
cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation);
if (!service) {
cbm_log_error("daemon.start_failed", "component", "runtime");
/* Recorded while the lifetime reservation is still held: a client
* that watches this generation vanish finds the cause already there
* and never launches a doomed replacement. */
host_start_failure_record(config->endpoint, "runtime");
}
cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation);
lifetime_reservation = NULL;
if (!service) {
cbm_log_error("daemon.start_failed", "component", "runtime");
host_state_free(&host);
host_log_close();
host_participant_guard_close(&participant_guard);
tests/test_daemon_bootstrap.c
+257
View File
@@ -1,18 +1,30 @@
/* RED contract for early process-role classification. */
#include "test_framework.h"
#include "test_helpers.h"
#include "daemon/bootstrap.h"
#include "daemon/host.h"
#include "daemon/ipc.h"
#include "daemon/ipc_internal.h"
#include "daemon/runtime.h"
#include "daemon/service.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
#include "foundation/platform.h"
#include <errno.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifndef _WIN32
#include <signal.h>
#include <sys/wait.h>
#include <unistd.h>
#endif
enum {
BOOTSTRAP_TEST_PATH_CAP = 1024,
@@ -874,6 +886,247 @@ TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous) {
}
#endif
#ifndef _WIN32
enum { BOOTSTRAP_ENOSPC_MAX_CHILDREN = 16, BOOTSTRAP_ENOSPC_LOG_CAP = 65536 };
typedef struct {
char parent[BOOTSTRAP_TEST_PATH_CAP];
cbm_daemon_build_identity_t identity;
pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN];
size_t child_count;
size_t spawn_calls;
} bootstrap_enospc_host_t;
/* The production spawn exec's the product binary; this one forks a REAL daemon
* host (cbm_daemon_host_run, the same entry `--cbm-daemon-internal` reaches)
* whose record publication fails with ENOSPC through the inherited seam. */
static bool bootstrap_enospc_host_spawn(void *opaque,
const cbm_daemon_bootstrap_launch_spec_t *spec) {
bootstrap_enospc_host_t *state = opaque;
if (!spec || !spec->detached) {
return false;
}
state->spawn_calls++;
if (state->child_count >= BOOTSTRAP_ENOSPC_MAX_CHILDREN) {
/* RED-run guard only: a client that keeps respawning a doomed daemon
* for its whole deadline must not fork without bound. */
return true;
}
pid_t child = fork();
if (child < 0) {
return false;
}
if (child == 0) {
cbm_daemon_ipc_endpoint_t *endpoint = cbm_daemon_bootstrap_endpoint_new(state->parent);
atomic_int stop_requested = ATOMIC_VAR_INIT(0);
cbm_daemon_host_config_t config = {
.endpoint = endpoint,
.identity = state->identity,
.executable_path = "/enospc-host-test",
.stop_requested = &stop_requested,
};
int run_result = endpoint ? cbm_daemon_host_run(&config) : 0;
_exit(run_result == -1 ? 0 : 50);
}
state->children[state->child_count++] = child;
return true;
}
static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) {
*nonzero_exits = 0;
for (size_t i = 0; i < state->child_count; i++) {
int status = 0;
pid_t waited;
do {
waited = waitpid(state->children[i], &status, WNOHANG);
} while (waited < 0 && errno == EINTR);
if (waited == 0) {
(void)kill(state->children[i], SIGKILL);
do {
waited = waitpid(state->children[i], &status, 0);
} while (waited < 0 && errno == EINTR);
}
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
(*nonzero_exits)++;
}
}
}
static bool bootstrap_read_file(const char *path, char *out, size_t capacity) {
FILE *file = cbm_fopen(path, "rb");
if (!file) {
return false;
}
size_t used = fread(out, 1, capacity - 1, file);
out[used] = '\0';
(void)fclose(file);
return true;
}
/* The record contract behind the fail-fast: a real listener failure in this
* process is recorded and read back with stage, errno, and path; a record
* older than the reader's spawn or for another endpoint is not evidence. */
TEST(daemon_bootstrap_start_failure_record_round_trip) {
bootstrap_endpoint_fixture_t fixture;
bootstrap_endpoint_fixture_t other;
ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "failure-record"));
char other_parent[BOOTSTRAP_TEST_PATH_CAP];
int other_written =
snprintf(other_parent, sizeof(other_parent), "%s/other-XXXXXX", fixture.parent);
ASSERT(other_written > 0 && other_written < (int)sizeof(other_parent));
ASSERT_TRUE(cbm_mkdtemp(other_parent) != NULL);
memset(&other, 0, sizeof(other));
other.endpoint = cbm_daemon_ipc_endpoint_new("1828000000000002", other_parent);
ASSERT_TRUE(other.endpoint != NULL);
char logs[BOOTSTRAP_TEST_PATH_CAP];
int logs_written = snprintf(logs, sizeof(logs), "%s/logs", fixture.parent);
ASSERT(logs_written > 0 && logs_written < (int)sizeof(logs));
char expected_path[BOOTSTRAP_TEST_PATH_CAP];
int path_written = snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp",
cbm_daemon_ipc_endpoint_address(fixture.endpoint));
ASSERT(path_written > 0 && path_written < (int)sizeof(expected_path));
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(fixture.endpoint);
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
ASSERT_TRUE(listener == NULL);
uint64_t now_s = (uint64_t)time(NULL);
bool recorded = cbm_daemon_bootstrap_start_failure_record(logs, fixture.endpoint, "runtime");
cbm_daemon_bootstrap_start_failure_t failure;
int found =
cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s - 2, &failure);
cbm_daemon_bootstrap_start_failure_t stale;
int stale_found =
cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s + 60, &stale);
cbm_daemon_bootstrap_start_failure_t foreign;
int foreign_found =
cbm_daemon_bootstrap_start_failure_read(logs, other.endpoint, now_s - 2, &foreign);
char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
cbm_daemon_bootstrap_start_failure_format(&failure, logs, message, sizeof(message));
cbm_daemon_ipc_endpoint_free(other.endpoint);
(void)th_rmtree(fixture.parent);
bootstrap_endpoint_fixture_finish(&fixture);
ASSERT_TRUE(recorded);
ASSERT_EQ(found, 1);
ASSERT_STR_EQ(failure.component, "runtime");
ASSERT_STR_EQ(failure.stage, "pending_publication");
ASSERT_EQ(failure.errno_value, ENOSPC);
ASSERT_STR_EQ(failure.path, expected_path);
ASSERT_TRUE(failure.pid == (uint64_t)getpid());
ASSERT_EQ(stale_found, 0);
ASSERT_EQ(foreign_found, 0);
ASSERT_TRUE(strstr(message, "CBM daemon failed to start: pending_publication failed with "
"ENOSPC (") != NULL);
ASSERT_TRUE(strstr(message, expected_path) != NULL);
ASSERT_TRUE(strstr(message, "/cbm-daemon.log") != NULL);
PASS();
}
/* #1828: a daemon that dies at publication (full /tmp) left every client
* waiting the full 30 s and then reporting "active or starting" -- the
* opposite of the truth. A real host is spawned against a runtime directory
* whose record writes fail with ENOSPC; the client must report "failed to
* start" naming the errno and the path, within seconds, not the deadline. */
TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) {
const char *old_cache = getenv("CBM_CACHE_DIR");
char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL;
bool snapshot_ok = !old_cache || saved_cache;
bootstrap_endpoint_fixture_t fixture;
bool fixture_ok = snapshot_ok && bootstrap_endpoint_fixture_start(&fixture, "enospc-host");
char cache[BOOTSTRAP_TEST_PATH_CAP] = {0};
char daemon_log[BOOTSTRAP_TEST_PATH_CAP] = {0};
char expected_path[BOOTSTRAP_TEST_PATH_CAP] = {0};
int cache_written =
fixture_ok ? snprintf(cache, sizeof(cache), "%s/cache", fixture.parent) : -1;
int log_written =
fixture_ok ? snprintf(daemon_log, sizeof(daemon_log), "%s/logs/cbm-daemon.log", cache) : -1;
int path_written = fixture_ok ? snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp",
cbm_daemon_ipc_endpoint_address(fixture.endpoint))
: -1;
bool environment_ready = cache_written > 0 && cache_written < (int)sizeof(cache) &&
log_written > 0 && log_written < (int)sizeof(daemon_log) &&
path_written > 0 && path_written < (int)sizeof(expected_path) &&
cbm_mkdir_p(cache, 0700) && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0;
char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE] = {0};
bool identity_ready = environment_ready && cbm_daemon_runtime_process_build_fingerprint(
(uint64_t)getpid(), self_build);
static bootstrap_enospc_host_t host;
memset(&host, 0, sizeof(host));
(void)snprintf(host.parent, sizeof(host.parent), "%s", fixture_ok ? fixture.parent : "");
host.identity = bootstrap_identity("2.4.0", self_build);
cbm_daemon_bootstrap_config_t config = {
.role = CBM_DAEMON_PROCESS_MCP_CLIENT,
.endpoint = fixture.endpoint,
.identity = &host.identity,
.executable_path = "/enospc-host-test",
.connect_timeout_ms = 200,
.startup_timeout_ms = 30000,
};
cbm_daemon_bootstrap_result_t result;
memset(&result, 0, sizeof(result));
cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED;
uint64_t elapsed_ms = UINT64_MAX;
if (identity_ready) {
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host);
uint64_t started = cbm_now_ms();
status = cbm_daemon_bootstrap_execute(&config, &result);
elapsed_ms = cbm_now_ms() - started;
cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL);
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
}
int nonzero_exits = 0;
bootstrap_enospc_reap(&host, &nonzero_exits);
static char log[BOOTSTRAP_ENOSPC_LOG_CAP];
log[0] = '\0';
bool log_read = bootstrap_read_file(daemon_log, log, sizeof(log));
const char *listen_failed = strstr(log, "msg=daemon.ipc.listen_failed");
bool daemon_named_cause = listen_failed && strstr(listen_failed, "errno=ENOSPC") != NULL &&
strstr(listen_failed, expected_path) != NULL;
bool message_names_failure = strstr(result.message, "failed to start") != NULL;
bool message_names_errno = strstr(result.message, "ENOSPC") != NULL;
bool message_names_path = strstr(result.message, expected_path) != NULL;
bool stale_wording = strstr(result.message, "active or starting") != NULL;
if (saved_cache) {
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1);
} else {
(void)cbm_unsetenv("CBM_CACHE_DIR");
}
free(saved_cache);
if (fixture_ok) {
(void)th_rmtree(fixture.parent);
bootstrap_endpoint_fixture_finish(&fixture);
}
ASSERT_TRUE(snapshot_ok);
ASSERT_TRUE(fixture_ok);
ASSERT_TRUE(environment_ready);
ASSERT_TRUE(identity_ready);
ASSERT_EQ(status, CBM_DAEMON_BOOTSTRAP_FAILED);
ASSERT_TRUE(result.daemon_spawned);
ASSERT_TRUE(host.child_count >= 1);
ASSERT_TRUE(log_read);
ASSERT_TRUE(daemon_named_cause);
ASSERT(elapsed_ms < 5000);
ASSERT_FALSE(stale_wording);
ASSERT_TRUE(message_names_failure);
ASSERT_TRUE(message_names_errno);
ASSERT_TRUE(message_names_path);
ASSERT_EQ(host.child_count, 1U);
ASSERT_EQ(nonzero_exits, 0);
bootstrap_endpoint_fixture_finish(&fixture);
PASS();
}
#endif
SUITE(daemon_bootstrap) {
RUN_TEST(daemon_bootstrap_classifies_default_and_ui_as_mcp_clients);
RUN_TEST(daemon_bootstrap_classifies_stateless_commands_without_client);
@@ -903,4 +1156,8 @@ SUITE(daemon_bootstrap) {
#ifdef __APPLE__
RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous);
#endif
#ifndef _WIN32
RUN_TEST(daemon_bootstrap_start_failure_record_round_trip);
RUN_TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication);
#endif
}