diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index 50f189036..0e4cbed61 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -6,13 +6,17 @@ #include "daemon/ipc.h" #include "daemon/service.h" #include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/log.h" #include "foundation/platform.h" +#include #include #include #include #include #include +#include #ifdef _WIN32 #ifndef WIN32_LEAN_AND_MEAN @@ -38,6 +42,14 @@ extern char **environ; enum { BOOTSTRAP_RETRY_NS = 1000000, + /* #1828: how often a waiting client re-reads the start-failure log + * (the wait loop itself ticks every millisecond). */ + BOOTSTRAP_START_FAILURE_CHECK_MS = 50, + /* A record this many seconds OLDER than the client's own spawn instant is + * still accepted: a sibling client's attempt that just failed is the same + * evidence, and second-granularity clocks need the slack anyway. */ + BOOTSTRAP_START_FAILURE_SKEW_S = 2, + BOOTSTRAP_START_FAILURE_LOG_CAP = 65536, BOOTSTRAP_COORDINATION_CLEANUP_MS = 500, BOOTSTRAP_PATH_CAP = 4096, }; @@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) { #endif } +static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log"; + +bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) { + if (!out || capacity == 0) { + return false; + } + out[0] = '\0'; + const char *cache = cbm_resolve_cache_dir(); + if (!cache || !cache[0]) { + return false; + } + int written = snprintf(out, capacity, "%s/logs", cache); + if (written <= 0 || (size_t)written >= capacity) { + out[0] = '\0'; + return false; + } + return true; +} + +/* One record field: the line format is tab-separated, so a tab or newline in + * a value (a path, in theory) becomes '?' rather than a parser ambiguity. */ +static void bootstrap_record_field(char *out, size_t capacity, const char *value) { + size_t used = 0; + for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) { + unsigned char ch = (unsigned char)*cursor; + out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch; + } + out[used] = '\0'; +} + +static uint64_t bootstrap_current_pid(void) { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)getpid(); +#endif +} + +bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component) { + if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) { + return false; + } + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint); + if (!runtime_dir || !runtime_dir[0]) { + return false; + } + cbm_daemon_ipc_listen_failure_t detail; + if (!cbm_daemon_ipc_listen_failure_detail(&detail)) { + memset(&detail, 0, sizeof(detail)); + } + char safe_runtime_dir[BOOTSTRAP_PATH_CAP]; + char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; + char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; + char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; + bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir); + bootstrap_record_field(safe_component, sizeof(safe_component), component); + bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage); + bootstrap_record_field(safe_path, sizeof(safe_path), detail.path); + FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME, + BOOTSTRAP_START_FAILURE_LOG_CAP); + if (!stream) { + return false; + } + int written = + fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL), + (unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component, + safe_stage, detail.errno_value, safe_path); + bool ok = written > 0 && fflush(stream) == 0; + if (fclose(stream) != 0) { + ok = false; + } + return ok; +} + +/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line + * (line is NUL-terminated, newline already stripped, and is modified). */ +static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out, + cbm_daemon_bootstrap_start_failure_t *out) { + char *fields[8]; + size_t count = 0; + char *cursor = line; + while (count < 8) { + fields[count++] = cursor; + char *tab = strchr(cursor, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + cursor = tab + 1; + } + if (count != 8 || strcmp(fields[0], "v1") != 0) { + return false; + } + memset(out, 0, sizeof(*out)); + out->recorded_at_s = strtoull(fields[1], NULL, 10); + out->pid = strtoull(fields[2], NULL, 10); + *runtime_dir_out = fields[3]; + (void)snprintf(out->component, sizeof(out->component), "%s", fields[4]); + (void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]); + out->errno_value = (int)strtol(fields[6], NULL, 10); + (void)snprintf(out->path, sizeof(out->path), "%s", fields[7]); + return true; +} + +static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) { + char path[BOOTSTRAP_PATH_CAP]; + int written = + snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME); + if (written <= 0 || written >= (int)sizeof(path)) { + *status_out = -1; + return NULL; + } +#ifndef _WIN32 + struct stat status; + if (lstat(path, &status) != 0) { + *status_out = errno == ENOENT ? 0 : -1; + return NULL; + } + if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) { + *status_out = -1; + return NULL; + } +#endif + /* The directory is the trust boundary (owner-only, no symlinked + * ancestry); the same validation guards cbm-daemon.log itself. */ + if (!cbm_daemon_ipc_private_directory_secure(log_directory)) { + *status_out = -1; + return NULL; + } + FILE *file = cbm_fopen(path, "rb"); + if (!file) { + *status_out = errno == ENOENT ? 0 : -1; + return NULL; + } + *status_out = 1; + return file; +} + +int cbm_daemon_bootstrap_start_failure_read(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure) { + if (!out_failure) { + return -1; + } + memset(out_failure, 0, sizeof(*out_failure)); + if (!log_directory || !log_directory[0] || !endpoint) { + return -1; + } + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint); + if (!runtime_dir || !runtime_dir[0]) { + return -1; + } + int status = 0; + FILE *file = bootstrap_start_failure_open(log_directory, &status); + if (!file) { + return status; + } + char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U); + if (!buffer) { + (void)fclose(file); + return -1; + } + size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file); + bool read_ok = !ferror(file); + (void)fclose(file); + buffer[used] = '\0'; + int found = 0; + if (read_ok) { + char *line = buffer; + while (line && *line) { + char *end = strchr(line, '\n'); + if (!end) { + break; /* a partial trailing line is a record still being written */ + } + *end = '\0'; + char *record_runtime_dir = NULL; + cbm_daemon_bootstrap_start_failure_t candidate; + if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) && + strcmp(record_runtime_dir, runtime_dir) == 0 && + candidate.recorded_at_s >= not_before_s) { + *out_failure = candidate; /* later lines are newer */ + found = 1; + } + line = end + 1; + } + } else { + found = -1; + } + free(buffer); + return found; +} + +void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure, + const char *log_directory, char *out, + size_t capacity) { + if (!out || capacity == 0) { + return; + } + if (!failure) { + out[0] = '\0'; + return; + } + const char *what = failure->stage[0] ? failure->stage + : failure->component[0] ? failure->component + : "startup"; + char where[BOOTSTRAP_PATH_CAP]; + if (log_directory && log_directory[0]) { + (void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory); + } else { + (void)snprintf(where, sizeof(where), "see the daemon log"); + } + if (failure->errno_value != 0 && failure->path[0]) { + (void)snprintf(out, capacity, + "CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what, + cbm_errno_name(failure->errno_value), strerror(failure->errno_value), + failure->path, where); + } else if (failure->errno_value != 0) { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s", + what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value), + where); + } else if (failure->path[0]) { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what, + failure->path, where); + } else { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where); + } +} + +/* After this attempt spawned a daemon: did that daemon (or a sibling attempt + * moments earlier) record a start failure? A found record ends the wait with + * its cause -- respawning a deterministically failing daemon until the + * deadline only produced "active or starting" (#1828). */ +static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config, + const cbm_daemon_bootstrap_ops_t *ops, + uint64_t spawn_wall_s, bool force, + uint64_t *next_check_ms, + cbm_daemon_bootstrap_result_t *result_out) { + if (spawn_wall_s == 0 || !ops->start_failure_probe) { + return false; + } + uint64_t now_ms = cbm_now_ms(); + if (!force && now_ms < *next_check_ms) { + return false; + } + *next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS; + uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S + ? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S + : 0; + cbm_daemon_bootstrap_start_failure_t failure; + if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) { + return false; + } + char logs[BOOTSTRAP_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) { + logs[0] = '\0'; + } + cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message, + sizeof(result_out->message)); + return true; +} + static void bootstrap_pause(uint64_t deadline) { uint64_t now = cbm_now_ms(); if (now >= deadline) { @@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( bool lock_acquired = false; bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED || probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL; + uint64_t spawn_wall_s = 0; + uint64_t next_failure_check_ms = 0; + bool start_failed = false; while (cbm_now_ms() < deadline) { + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } if (!bootstrap_probe_is_waitable(probe)) { break; } @@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( break; } + /* Never relaunch a daemon whose predecessor already recorded why it + * could not start; the record is written before that daemon lets go + * of its lifetime reservation, so this unconditional read sees it. */ + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } cbm_daemon_bootstrap_launch_spec_t spec; bool spec_ready = config->spawn_permanent ? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec) : cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec); + spawn_wall_s = (uint64_t)time(NULL); if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) || !ops->spawn_daemon(ops->context, &spec)) { probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR; @@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( if (!bootstrap_probe_is_waitable(probe)) { break; } + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } } while (cbm_now_ms() < deadline); + if (start_failed) { + break; + } if (!lock_acquired) { continue; } @@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( } result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED; - if (muted_holder_pid != 0) { + if (start_failed) { + /* result_out->message already names the recorded cause. */ + } else if (muted_holder_pid != 0) { /* The one diagnostic the 2026-08-29 zombie recovery had to assemble by * hand from process, pipe, and log correlation: name the pid that * holds the endpoint without answering, and say what to do with it. */ @@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock return released; } +static int bootstrap_production_start_failure_probe( + void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure) { + (void)context; + char logs[BOOTSTRAP_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) { + return -1; + } + return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure); +} + static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) { (void)context; return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock); @@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message) (void)fflush(stderr); } +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test; +static void *g_bootstrap_spawn_override_context_for_test; + +void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn, + void *context) { + g_bootstrap_spawn_override_context_for_test = context; + g_bootstrap_spawn_override_for_test = spawn; +} + +static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) { + (void)context; + return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec); +} +#endif + cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) { bootstrap_production_context_t context = {0}; - const cbm_daemon_bootstrap_ops_t ops = { + cbm_daemon_bootstrap_ops_t ops = { .context = &context, .cohort_acquire = bootstrap_production_cohort_acquire, .cohort_release = bootstrap_production_cohort_release, @@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( .startup_lock_release = bootstrap_production_unlock, .spawn_daemon = bootstrap_production_spawn, .visible_diagnostic = bootstrap_production_diagnostic, + .start_failure_probe = bootstrap_production_start_failure_probe, }; +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_bootstrap_spawn_override_for_test) { + ops.spawn_daemon = bootstrap_seam_spawn; + } +#endif return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out); } diff --git a/src/daemon/bootstrap.h b/src/daemon/bootstrap.h index acb6c657b..8b1e797dd 100644 --- a/src/daemon/bootstrap.h +++ b/src/daemon/bootstrap.h @@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect typedef void *cbm_daemon_bootstrap_lock_t; typedef void *cbm_daemon_bootstrap_cohort_t; +/* Durable daemon start-failure record (#1828). A detached daemon has no + * stderr and no channel back to the client that launched it; when its + * listener publication fails (a full /tmp, for instance) the client used to + * respawn it for the whole startup deadline and then report "active or + * starting". The host now appends one record per failed start to + * /logs/cbm-daemon.start-failures.log -- the same owner-only log + * directory as cbm-daemon.log, deliberately NOT the runtime directory: that + * one is exactly what a full /tmp cannot write, and a new artifact there + * would entangle stale-generation cleanup. A waiting client reads the newest + * record for its endpoint that is not older than its own spawn and fails + * immediately with the cause. Diagnostic only: policy never reads it. */ +enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 }; +typedef struct { + uint64_t recorded_at_s; /* wall clock, seconds */ + uint64_t pid; + char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */ + char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */ + int errno_value; /* 0 when unknown */ + char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */ +} cbm_daemon_bootstrap_start_failure_t; + +/* /logs, the owner-only directory that holds every daemon log. */ +bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity); +/* Append one record for this process's most recent listener failure detail + * (cbm_daemon_ipc_listen_failure_detail) under `component`. */ +bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component); +/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1 + * and fills out_failure, 0 when none matches, -1 when the log directory or + * file cannot be validated. */ +int cbm_daemon_bootstrap_start_failure_read(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure); +/* The client-facing sentence: "CBM daemon failed to start: failed + * with ENOSPC (No space left on device) at ; see ". */ +void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure, + const char *log_directory, char *out, + size_t capacity); + /* Injectable OS/runtime boundary used by the deterministic unit contract. * Production callers use cbm_daemon_bootstrap_execute(), whose built-in * operations delegate to daemon IPC/runtime and write visible diagnostics to @@ -149,6 +190,12 @@ typedef struct { bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io); bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec); void (*visible_diagnostic)(void *context, const char *message); + /* Optional. After this attempt spawned a daemon, report a start-failure + * record for the endpoint not older than not_before_s (1 found, 0 none, + * -1 unreadable). A found record ends the wait immediately (#1828). */ + int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure); } cbm_daemon_bootstrap_ops_t; cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( @@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( /* Test seam for the same state machine. All callbacks are synchronous and * borrowed for the duration of the call. */ +#ifdef CBM_ENABLE_TEST_SEAMS +/* Replace ONLY the production spawn while every other production operation + * (cohort, probe, startup lock, handoff) stays real. A test forks a real + * daemon host in place of exec'ing the product binary, so the complete + * client/daemon rendezvous runs in one deterministic process tree (#1828). */ +typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context, + const cbm_daemon_bootstrap_launch_spec_t *spec); +void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn, + void *context); +#endif + cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops, cbm_daemon_bootstrap_result_t *result_out); diff --git a/src/daemon/host.c b/src/daemon/host.c index f475658b8..dfab03e13 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -5,6 +5,7 @@ #include "daemon/host_internal.h" #include "daemon/application.h" +#include "daemon/bootstrap.h" #include "daemon/runtime.h" #include "daemon/project_lock.h" #include "daemon/version_cohort.h" @@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) { return true; } +/* #1828: a detached daemon has no stderr; its start failure must reach the + * client that is waiting for it, or that client burns its whole deadline and + * reports the opposite of the truth. */ +static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component) { + char logs[HOST_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) || + !cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) { + cbm_log_error("daemon.start_failure_record_failed", "component", component); + } +} + static void host_log_close(void) { cbm_log_set_sink(NULL); if (g_host_log_mutex_initialized) { @@ -1051,10 +1064,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) { }; cbm_daemon_runtime_service_t *service = cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation); + if (!service) { + cbm_log_error("daemon.start_failed", "component", "runtime"); + /* Recorded while the lifetime reservation is still held: a client + * that watches this generation vanish finds the cause already there + * and never launches a doomed replacement. */ + host_start_failure_record(config->endpoint, "runtime"); + } cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation); lifetime_reservation = NULL; if (!service) { - cbm_log_error("daemon.start_failed", "component", "runtime"); host_state_free(&host); host_log_close(); host_participant_guard_close(&participant_guard); diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index a9c8a368e..20213ef5b 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -1,18 +1,30 @@ /* RED contract for early process-role classification. */ #include "test_framework.h" +#include "test_helpers.h" #include "daemon/bootstrap.h" +#include "daemon/host.h" #include "daemon/ipc.h" +#include "daemon/ipc_internal.h" +#include "daemon/runtime.h" #include "daemon/service.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" #include "foundation/platform.h" +#include #include #include #include +#include #include +#include +#ifndef _WIN32 +#include +#include +#include +#endif enum { BOOTSTRAP_TEST_PATH_CAP = 1024, @@ -874,6 +886,247 @@ TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous) { } #endif +#ifndef _WIN32 +enum { BOOTSTRAP_ENOSPC_MAX_CHILDREN = 16, BOOTSTRAP_ENOSPC_LOG_CAP = 65536 }; + +typedef struct { + char parent[BOOTSTRAP_TEST_PATH_CAP]; + cbm_daemon_build_identity_t identity; + pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + size_t child_count; + size_t spawn_calls; +} bootstrap_enospc_host_t; + +/* The production spawn exec's the product binary; this one forks a REAL daemon + * host (cbm_daemon_host_run, the same entry `--cbm-daemon-internal` reaches) + * whose record publication fails with ENOSPC through the inherited seam. */ +static bool bootstrap_enospc_host_spawn(void *opaque, + const cbm_daemon_bootstrap_launch_spec_t *spec) { + bootstrap_enospc_host_t *state = opaque; + if (!spec || !spec->detached) { + return false; + } + state->spawn_calls++; + if (state->child_count >= BOOTSTRAP_ENOSPC_MAX_CHILDREN) { + /* RED-run guard only: a client that keeps respawning a doomed daemon + * for its whole deadline must not fork without bound. */ + return true; + } + pid_t child = fork(); + if (child < 0) { + return false; + } + if (child == 0) { + cbm_daemon_ipc_endpoint_t *endpoint = cbm_daemon_bootstrap_endpoint_new(state->parent); + atomic_int stop_requested = ATOMIC_VAR_INIT(0); + cbm_daemon_host_config_t config = { + .endpoint = endpoint, + .identity = state->identity, + .executable_path = "/enospc-host-test", + .stop_requested = &stop_requested, + }; + int run_result = endpoint ? cbm_daemon_host_run(&config) : 0; + _exit(run_result == -1 ? 0 : 50); + } + state->children[state->child_count++] = child; + return true; +} + +static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) { + *nonzero_exits = 0; + for (size_t i = 0; i < state->child_count; i++) { + int status = 0; + pid_t waited; + do { + waited = waitpid(state->children[i], &status, WNOHANG); + } while (waited < 0 && errno == EINTR); + if (waited == 0) { + (void)kill(state->children[i], SIGKILL); + do { + waited = waitpid(state->children[i], &status, 0); + } while (waited < 0 && errno == EINTR); + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + (*nonzero_exits)++; + } + } +} + +static bool bootstrap_read_file(const char *path, char *out, size_t capacity) { + FILE *file = cbm_fopen(path, "rb"); + if (!file) { + return false; + } + size_t used = fread(out, 1, capacity - 1, file); + out[used] = '\0'; + (void)fclose(file); + return true; +} + +/* The record contract behind the fail-fast: a real listener failure in this + * process is recorded and read back with stage, errno, and path; a record + * older than the reader's spawn or for another endpoint is not evidence. */ +TEST(daemon_bootstrap_start_failure_record_round_trip) { + bootstrap_endpoint_fixture_t fixture; + bootstrap_endpoint_fixture_t other; + ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "failure-record")); + char other_parent[BOOTSTRAP_TEST_PATH_CAP]; + int other_written = + snprintf(other_parent, sizeof(other_parent), "%s/other-XXXXXX", fixture.parent); + ASSERT(other_written > 0 && other_written < (int)sizeof(other_parent)); + ASSERT_TRUE(cbm_mkdtemp(other_parent) != NULL); + memset(&other, 0, sizeof(other)); + other.endpoint = cbm_daemon_ipc_endpoint_new("1828000000000002", other_parent); + ASSERT_TRUE(other.endpoint != NULL); + char logs[BOOTSTRAP_TEST_PATH_CAP]; + int logs_written = snprintf(logs, sizeof(logs), "%s/logs", fixture.parent); + ASSERT(logs_written > 0 && logs_written < (int)sizeof(logs)); + char expected_path[BOOTSTRAP_TEST_PATH_CAP]; + int path_written = snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp", + cbm_daemon_ipc_endpoint_address(fixture.endpoint)); + ASSERT(path_written > 0 && path_written < (int)sizeof(expected_path)); + + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(fixture.endpoint); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + ASSERT_TRUE(listener == NULL); + + uint64_t now_s = (uint64_t)time(NULL); + bool recorded = cbm_daemon_bootstrap_start_failure_record(logs, fixture.endpoint, "runtime"); + cbm_daemon_bootstrap_start_failure_t failure; + int found = + cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s - 2, &failure); + cbm_daemon_bootstrap_start_failure_t stale; + int stale_found = + cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s + 60, &stale); + cbm_daemon_bootstrap_start_failure_t foreign; + int foreign_found = + cbm_daemon_bootstrap_start_failure_read(logs, other.endpoint, now_s - 2, &foreign); + char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE]; + cbm_daemon_bootstrap_start_failure_format(&failure, logs, message, sizeof(message)); + + cbm_daemon_ipc_endpoint_free(other.endpoint); + (void)th_rmtree(fixture.parent); + bootstrap_endpoint_fixture_finish(&fixture); + + ASSERT_TRUE(recorded); + ASSERT_EQ(found, 1); + ASSERT_STR_EQ(failure.component, "runtime"); + ASSERT_STR_EQ(failure.stage, "pending_publication"); + ASSERT_EQ(failure.errno_value, ENOSPC); + ASSERT_STR_EQ(failure.path, expected_path); + ASSERT_TRUE(failure.pid == (uint64_t)getpid()); + ASSERT_EQ(stale_found, 0); + ASSERT_EQ(foreign_found, 0); + ASSERT_TRUE(strstr(message, "CBM daemon failed to start: pending_publication failed with " + "ENOSPC (") != NULL); + ASSERT_TRUE(strstr(message, expected_path) != NULL); + ASSERT_TRUE(strstr(message, "/cbm-daemon.log") != NULL); + PASS(); +} + +/* #1828: a daemon that dies at publication (full /tmp) left every client + * waiting the full 30 s and then reporting "active or starting" -- the + * opposite of the truth. A real host is spawned against a runtime directory + * whose record writes fail with ENOSPC; the client must report "failed to + * start" naming the errno and the path, within seconds, not the deadline. */ +TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) { + const char *old_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL; + bool snapshot_ok = !old_cache || saved_cache; + + bootstrap_endpoint_fixture_t fixture; + bool fixture_ok = snapshot_ok && bootstrap_endpoint_fixture_start(&fixture, "enospc-host"); + char cache[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char daemon_log[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char expected_path[BOOTSTRAP_TEST_PATH_CAP] = {0}; + int cache_written = + fixture_ok ? snprintf(cache, sizeof(cache), "%s/cache", fixture.parent) : -1; + int log_written = + fixture_ok ? snprintf(daemon_log, sizeof(daemon_log), "%s/logs/cbm-daemon.log", cache) : -1; + int path_written = fixture_ok ? snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp", + cbm_daemon_ipc_endpoint_address(fixture.endpoint)) + : -1; + bool environment_ready = cache_written > 0 && cache_written < (int)sizeof(cache) && + log_written > 0 && log_written < (int)sizeof(daemon_log) && + path_written > 0 && path_written < (int)sizeof(expected_path) && + cbm_mkdir_p(cache, 0700) && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; + + char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE] = {0}; + bool identity_ready = environment_ready && cbm_daemon_runtime_process_build_fingerprint( + (uint64_t)getpid(), self_build); + static bootstrap_enospc_host_t host; + memset(&host, 0, sizeof(host)); + (void)snprintf(host.parent, sizeof(host.parent), "%s", fixture_ok ? fixture.parent : ""); + host.identity = bootstrap_identity("2.4.0", self_build); + + cbm_daemon_bootstrap_config_t config = { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = fixture.endpoint, + .identity = &host.identity, + .executable_path = "/enospc-host-test", + .connect_timeout_ms = 200, + .startup_timeout_ms = 30000, + }; + cbm_daemon_bootstrap_result_t result; + memset(&result, 0, sizeof(result)); + cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED; + uint64_t elapsed_ms = UINT64_MAX; + if (identity_ready) { + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host); + uint64_t started = cbm_now_ms(); + status = cbm_daemon_bootstrap_execute(&config, &result); + elapsed_ms = cbm_now_ms() - started; + cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + } + int nonzero_exits = 0; + bootstrap_enospc_reap(&host, &nonzero_exits); + + static char log[BOOTSTRAP_ENOSPC_LOG_CAP]; + log[0] = '\0'; + bool log_read = bootstrap_read_file(daemon_log, log, sizeof(log)); + const char *listen_failed = strstr(log, "msg=daemon.ipc.listen_failed"); + bool daemon_named_cause = listen_failed && strstr(listen_failed, "errno=ENOSPC") != NULL && + strstr(listen_failed, expected_path) != NULL; + bool message_names_failure = strstr(result.message, "failed to start") != NULL; + bool message_names_errno = strstr(result.message, "ENOSPC") != NULL; + bool message_names_path = strstr(result.message, expected_path) != NULL; + bool stale_wording = strstr(result.message, "active or starting") != NULL; + + if (saved_cache) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache); + if (fixture_ok) { + (void)th_rmtree(fixture.parent); + bootstrap_endpoint_fixture_finish(&fixture); + } + + ASSERT_TRUE(snapshot_ok); + ASSERT_TRUE(fixture_ok); + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(identity_ready); + ASSERT_EQ(status, CBM_DAEMON_BOOTSTRAP_FAILED); + ASSERT_TRUE(result.daemon_spawned); + ASSERT_TRUE(host.child_count >= 1); + ASSERT_TRUE(log_read); + ASSERT_TRUE(daemon_named_cause); + ASSERT(elapsed_ms < 5000); + ASSERT_FALSE(stale_wording); + ASSERT_TRUE(message_names_failure); + ASSERT_TRUE(message_names_errno); + ASSERT_TRUE(message_names_path); + ASSERT_EQ(host.child_count, 1U); + ASSERT_EQ(nonzero_exits, 0); + bootstrap_endpoint_fixture_finish(&fixture); + PASS(); +} +#endif + SUITE(daemon_bootstrap) { RUN_TEST(daemon_bootstrap_classifies_default_and_ui_as_mcp_clients); RUN_TEST(daemon_bootstrap_classifies_stateless_commands_without_client); @@ -903,4 +1156,8 @@ SUITE(daemon_bootstrap) { #ifdef __APPLE__ RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous); #endif +#ifndef _WIN32 + RUN_TEST(daemon_bootstrap_start_failure_record_round_trip); + RUN_TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication); +#endif }