diff --git a/README.md b/README.md index 4466dae50..c34d20b98 100644 --- a/README.md +++ b/README.md @@ -928,8 +928,8 @@ codebase-memory-mcp config set watcher_enabled false # stop the watcher thre codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit -codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit -codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree charged-memory limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional per-request duration codebase-memory-mcp config reset auto_index # reset to default ``` diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3d3e9a971..a02459b4a 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -93,8 +93,8 @@ Current keys: | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | -| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | -| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | +| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | +| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 66be4ecf4..152599360 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -27,24 +27,30 @@ ranges are rejected without changing the stored value. | Key | Default | Accepted value | Protects | |---|---:|---:|---| -| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | -| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request | ```bash codebase-memory-mcp config set index_max_rss_mb 8192 codebase-memory-mcp config set index_max_duration_seconds 3600 ``` -RSS is the current resident memory of the contained worker and every descendant, -not the worker's allocation budget and not peak memory. This hard watchdog is +RSS is the charged memory of the contained worker and every descendant, not +the worker's allocation budget and not peak memory. On macOS that quantity is +`phys_footprint` (the same number `cbm_mem_charged()` enforces), not +`resident_size`, which still counts pages the allocator has already handed +back. Linux and Windows use RSS / working set. This hard watchdog is separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor -samples RSS at most once every 250 milliseconds so the watchdog does not turn -full process-table enumeration into a busy loop. +samples the tree at most once every 250 milliseconds so the watchdog does not +turn full process-table enumeration into a busy loop. -Duration uses a monotonic clock from successful spawn. It is independent of the -existing 15-minute quiet timeout: continuous log progress does not reset total -duration, while the quiet timeout continues to identify a worker that stops -making progress. +Duration is per request: the clock starts at the first worker spawn and is +not reset when crash/hang recovery starts a later attempt. Continuous log +progress does not reset it. It is independent of the existing 15-minute quiet +timeout, which still identifies a worker that stops making progress. A +duration limit shorter than that quiet timeout kills a hung worker before hang +quarantine can name the in-flight file, so the next attempt may hang on the +same file. Equality is allowed. The first RSS or elapsed-duration observation above its limit starts the existing graceful-to-force process-tree shutdown. CBM reports diff --git a/src/cli/cli.c b/src/cli/cli.c index 00b2db341..fd51c48e9 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7603,8 +7603,9 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, - {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, - {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree charged memory MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", + "Max wall-clock seconds for the whole index request, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing diff --git a/src/daemon/application.c b/src/daemon/application.c index a7b2f4f5b..0f696d59c 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -159,6 +159,7 @@ struct cbm_daemon_application_job { * job outlives the request and session that started it. */ bool async_owned; bool worker_started; + uint64_t request_started_ms; cbm_daemon_application_job_t *next; }; @@ -361,6 +362,7 @@ static void application_worker_diagnostic(const char *line, void *context) { static int application_worker_start_default(void *context, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { (void)context; cbm_index_resource_policy_t resource_policy; @@ -382,7 +384,7 @@ static int application_worker_start_default(void *context, const char *args_json yyjson_doc_free(args_doc); int result = cbm_index_worker_start_with_policy_log( args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, - application_worker_diagnostic, worker, &worker->handle); + application_worker_diagnostic, worker, duration_origin_ms, &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; @@ -1377,10 +1379,13 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli } cbm_daemon_application_worker_t worker = NULL; + if (job->request_started_ms == 0) { + job->request_started_ms = cbm_index_worker_now_ms(); + } application_tmp_lock(); - int start_result = - application->worker_ops.start(application->worker_ops.context, job->args_json, - memory_budget_bytes, marker_path, quarantine_path, &worker); + int start_result = application->worker_ops.start( + application->worker_ops.context, job->args_json, memory_budget_bytes, marker_path, + quarantine_path, job->request_started_ms, &worker); application_tmp_unlock(); if (start_result != 0 || !worker) { return application_job_cancel_requested(job) ? APPLICATION_ATTEMPT_CANCELLED diff --git a/src/daemon/application.h b/src/daemon/application.h index 933186aa7..426bef3db 100644 --- a/src/daemon/application.h +++ b/src/daemon/application.h @@ -31,7 +31,7 @@ typedef void *cbm_daemon_application_update_worker_t; typedef struct { void *context; int (*start)(void *context, const char *args_json, size_t memory_budget_bytes, - const char *marker_file, const char *quarantine_file, + const char *marker_file, const char *quarantine_file, uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out); cbm_index_worker_poll_t (*poll)(void *context, cbm_daemon_application_worker_t worker, const cbm_index_worker_result_t **result_out); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 474ada1d2..519a554e2 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -513,6 +513,12 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces bool root_failed = false; for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { DWORD pid = (DWORD)processes->ProcessIdList[index]; + /* GetProcessMemoryInfo documents PROCESS_QUERY_INFORMATION | + * PROCESS_VM_READ. PROCESS_QUERY_LIMITED_INFORMATION is a lesser + * alternative on newer Windows; the Job Object membership list + * already closes most of the pid-reuse window between listing and + * opening. A member we cannot open is a probe failure when it is + * the root. */ HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); if (!member) { root_failed = root_failed || pid == process->process_id; @@ -592,7 +598,11 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces root_failed = root_failed || pids[index] == process->pid; continue; } - cbm_rss_add_saturated(&total, usage.ri_resident_size); + /* phys_footprint is what the OS charges, matching cbm_mem_charged(). + * ri_resident_size still counts pages mimalloc already marked + * MADV_FREE_REUSABLE, so a Mac worker can look like 17 GB RSS while + * the kernel holds it to 5.5 GB. */ + cbm_rss_add_saturated(&total, usage.ri_phys_footprint); measured++; } cbm_free(CBM_MEM_CLASS_OTHER, pids); diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index bfcb2311b..94a8a2ed0 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -155,10 +155,12 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); -/* Read the current resident-set size of the complete contained process tree. - * OK returns an overflow-safe byte total, EMPTY means the owned tree currently - * has no observable members, and ERROR means no trustworthy measurement could - * be obtained. Individual processes that exit during enumeration are ignored. */ +/* Read the current charged memory of the complete contained process tree. + * On macOS this is phys_footprint (what the OS holds against the process); + * elsewhere it is RSS. OK returns an overflow-safe byte total, EMPTY means + * the owned tree currently has no observable members, and ERROR means no + * trustworthy measurement could be obtained. Individual processes that exit + * during enumeration are ignored. */ cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, uint64_t *rss_bytes); bool cbm_subprocess_root_running(const cbm_subprocess_t *process); diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 4ffeecbb8..e84dff755 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -881,7 +881,8 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { + void *log_context, uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -985,34 +986,42 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt return -1; } if (handle->resource_policy.max_duration_ms.enabled) { - handle->started_ms = worker_resource_now_ms(); + /* Duration is per request, not per spawn: a crash/hang recovery must + * not reset the clock. Callers pass the request origin; 0 means now. */ + uint64_t origin = duration_origin_ms != 0 ? duration_origin_ms : worker_resource_now_ms(); + handle->started_ms = origin; } *handle_out = handle; return 0; } +uint64_t cbm_index_worker_now_ms(void) { + return worker_resource_now_ms(); +} + int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, log_callback, log_context, handle_out); + quarantine_file, log_callback, log_context, 0, handle_out); } int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, NULL, NULL, handle_out); + quarantine_file, NULL, NULL, 0, handle_out); } int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, - const char *quarantine_file, + const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + marker_file, quarantine_file, NULL, NULL, duration_origin_ms, + handle_out); } int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, @@ -1020,10 +1029,11 @@ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_ bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, marker_file, quarantine_file, log_callback, log_context, - handle_out); + duration_origin_ms, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -1167,14 +1177,15 @@ static int worker_spawn_internal(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { + uint64_t duration_origin_ms, cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, - quarantine_file, log_callback, log_context, &handle) != 0) { + quarantine_file, log_callback, log_context, duration_origin_ms, + &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -1206,16 +1217,17 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, - log_callback, log_context, cancel_requested, result); + log_callback, log_context, cancel_requested, 0, result); } int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result) { return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, - result); + duration_origin_ms, result); } int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index a86761d77..2cb244a7e 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -186,19 +186,24 @@ int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bo int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, - const char *quarantine_file, + const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); /* Policy plus the request-scoped log callback. The daemon default start path * needs both: a missing policy fails closed, and worker diagnostics still - * reach the daemon journal. */ + * reach the daemon journal. duration_origin_ms is the request clock; 0 means now. */ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); +/* Supervisor clock used to stamp a request-scoped duration origin. Tests may + * replace it; production is cbm_now_ms(). */ +uint64_t cbm_index_worker_now_ms(void); + /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context * remains caller-owned until terminal. No process-global sink is installed. @@ -264,7 +269,8 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1016f781a..f23a00e5c 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11617,12 +11617,14 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); + uint64_t request_started_ms = cbm_index_worker_now_ms(); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; int rc = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, - srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); + srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, + request_started_ms, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -11704,7 +11706,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wr2); + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (wr2.response_missing) { @@ -11831,7 +11833,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, int rcp = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wrp); + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = cbm_mcp_supervised_result_disposition(rcp, &wrp); if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 5fe4e618f..5641ecbac 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1703,7 +1703,9 @@ static void app_fake_worker_read_file(const char *path, char *out, size_t out_si static int app_fake_worker_start(void *opaque, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { + (void)duration_origin_ms; app_fake_worker_context_t *context = opaque; app_fake_worker_t *worker = calloc(1, sizeof(*worker)); if (!worker) { diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 88d58c0d7..df55407da 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -689,8 +689,8 @@ TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat) { (void)cbm_unsetenv("CBM_LOG_LEVEL"); cbm_index_worker_handle_t *handle = NULL; - int start_rc = cbm_index_worker_start("{\"__cbm_test_worker\":\"heartbeat\"}", 0, false, - NULL, NULL, &handle); + int start_rc = cbm_index_worker_start("{\"__cbm_test_worker\":\"heartbeat\"}", 0, false, NULL, + NULL, &handle); char log_path[INDEX_SUPERVISOR_TEST_PATH_CAP] = {0}; if (handle) { (void)snprintf(log_path, sizeof(log_path), "%s", cbm_index_worker_log_path(handle)); @@ -698,8 +698,8 @@ TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat) { bool ready = log_path[0] && index_supervisor_test_wait_file_text( log_path, "async worker heartbeat probe ready", INDEX_SUPERVISOR_TEST_READY_MS); - bool heartbeat = ready && index_supervisor_test_wait_file_text( - log_path, "msg=pipeline.discover", 1000); + bool heartbeat = + ready && index_supervisor_test_wait_file_text(log_path, "msg=pipeline.discover", 1000); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1034,8 +1034,9 @@ static uint64_t index_supervisor_fake_clock(void *context) { return fake->now_ms; } -static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, - uint64_t *rss_bytes, void *context) { +static cbm_proc_tree_rss_status_t index_supervisor_fake_charged_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context) { (void)process; index_supervisor_resource_fake_t *fake = context; int value_index = @@ -1065,11 +1066,11 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1092,7 +1093,7 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { PASS(); } -TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { +TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree) { const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; index_supervisor_resource_fake_t fake = { .now_ms = 100, @@ -1101,11 +1102,11 @@ TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t equal_state = handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; @@ -1141,11 +1142,11 @@ TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { .rss_status = CBM_PROC_TREE_RSS_EMPTY, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); fake.now_ms = 1100; const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t equal_state = @@ -1171,6 +1172,60 @@ TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { PASS(); } +TEST(index_supervisor_duration_spans_recovery_attempts) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *first = NULL; + int first_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &first); + fake.now_ms = 500; + const cbm_index_worker_result_t *first_result = NULL; + cbm_index_worker_poll_t mid = + first ? cbm_index_worker_poll(first, &first_result) : CBM_INDEX_WORKER_POLL_ERROR; + bool cancel_accepted = first && cbm_index_worker_request_cancel(first); + bool first_terminal = first && index_supervisor_test_poll_terminal( + first, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &first_result); + if (first_terminal) { + cbm_index_worker_destroy(first); + first = NULL; + } else { + index_supervisor_test_cleanup_handle(first); + first = NULL; + } + + fake.now_ms = 1101; + cbm_index_worker_handle_t *second = NULL; + int second_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &second); + const cbm_index_worker_result_t *second_result = NULL; + bool second_terminal = second && index_supervisor_test_poll_terminal( + second, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &second_result); + bool limited = second_terminal && second_result && + second_result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + second_result->resource_violation.observed == 1001 && + second_result->resource_violation.limit == 1000 && + second_result->outcome != CBM_PROC_HANG && second_result->tree_quiesced; + if (second_terminal) { + cbm_index_worker_destroy(second); + } else { + index_supervisor_test_cleanup_handle(second); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(first_rc, 0); + ASSERT_EQ(mid, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(first_terminal); + ASSERT_EQ(second_rc, 0); + ASSERT_TRUE(limited); + PASS(); +} + TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; @@ -1179,7 +1234,7 @@ TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1208,11 +1263,11 @@ TEST(index_supervisor_cancel_precedes_resource_probe) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); fake.now_ms = 2000; bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); const cbm_index_worker_result_t *result = NULL; @@ -1242,11 +1297,11 @@ TEST(index_supervisor_three_failed_rss_probes_fail_closed) { .rss_status = CBM_PROC_TREE_RSS_ERROR, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t first = handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; @@ -1294,8 +1349,9 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); RUN_TEST(index_supervisor_disabled_limits_do_not_probe); - RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree); RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_duration_spans_recovery_attempts); RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); RUN_TEST(index_supervisor_cancel_precedes_resource_probe); RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed);