mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
feat(diagnostics): allocation-site profiler for the #581 analysis harness
The census could say which POOL held memory but never which allocation put it there, so each hypothesis needed a code change plus an eight-minute soak to test and the investigation could only ask one yes/no question at a time. Records allocations at or above a threshold (default 32 KiB, since arena fragmentation is driven by large transient runs) against their captured stack, into fixed tables whose exhaustion is counted and reported rather than silently dropped. Emitted alongside every census sample, so pool totals and attribution always describe the same instant, plus an explicit unattributed remainder so incomplete coverage announces itself. scripts/memlab-report.py ranks sites by retained-byte growth using a linear fit and can diff two platforms' profiles against each other. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
1 parent
5929f6a2e6
commit
7a2376987e
6 files changed
+525
-4
No files matched your search
@@ -140,6 +140,7 @@ FOUNDATION_SRCS = \
|
||||
src/foundation/compat_fs.c \
|
||||
src/foundation/compat_regex.c \
|
||||
src/foundation/mem.c \
|
||||
src/foundation/mem_profile.c \
|
||||
src/foundation/diagnostics.c \
|
||||
src/foundation/profile.c \
|
||||
src/foundation/dump_verify.c \
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Analyse #581 memory-lab output: attribute growth to allocation sites.
|
||||
|
||||
Reads the JSONL the profiler appends (CBM_MEM_PROFILE_OUT) and the mem.census
|
||||
lines from a daemon log, and prints:
|
||||
|
||||
* retention per request, as a linear fit rather than first-vs-last, so noise
|
||||
does not masquerade as a trend
|
||||
* the call sites whose live bytes grow across the run, ranked by growth
|
||||
* an explicit unattributed remainder: process growth the profiler could not
|
||||
account for. That number is the honesty check — a report that attributed
|
||||
everything while tracking only >=32 KiB allocations would be lying.
|
||||
|
||||
Usage:
|
||||
memlab-report.py <profile.jsonl> [--census <daemon.log>] [--top N]
|
||||
memlab-report.py <a.jsonl> --diff <b.jsonl> # platform comparison
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
def load_profile(path):
|
||||
"""Return (summaries, site_series). Records are appended over time, so the
|
||||
Nth occurrence of a site is its Nth sample."""
|
||||
summaries, series = [], defaultdict(list)
|
||||
with open(path, "r", errors="replace") as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if "site" in record:
|
||||
series[record["site"]].append(record)
|
||||
else:
|
||||
summaries.append(record)
|
||||
return summaries, series
|
||||
|
||||
|
||||
def linear_slope(values):
|
||||
"""Least-squares slope over sample index. Resistant to the single-outlier
|
||||
reading that a first-vs-last delta would treat as the whole trend."""
|
||||
n = len(values)
|
||||
if n < 2:
|
||||
return 0.0
|
||||
mean_x = (n - 1) / 2.0
|
||||
mean_y = sum(values) / n
|
||||
num = sum((i - mean_x) * (v - mean_y) for i, v in enumerate(values))
|
||||
den = sum((i - mean_x) ** 2 for i in range(n))
|
||||
return num / den if den else 0.0
|
||||
|
||||
|
||||
def parse_census(path):
|
||||
"""Pull the pool totals out of mem.census log lines."""
|
||||
fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb", "prof_live_kb")
|
||||
rows = []
|
||||
pattern = re.compile(r"(\w+)=(\S+)")
|
||||
with open(path, "r", errors="replace") as handle:
|
||||
for line in handle:
|
||||
if "mem.census" not in line:
|
||||
continue
|
||||
kv = dict(pattern.findall(line))
|
||||
if not any(f in kv for f in fields):
|
||||
continue
|
||||
rows.append({f: int(kv[f]) for f in fields if f in kv and kv[f].isdigit()})
|
||||
return rows
|
||||
|
||||
|
||||
def report(path, census_path, top):
|
||||
summaries, series = load_profile(path)
|
||||
if not summaries and not series:
|
||||
print(f"{path}: no profiler records — was CBM_MEM_PROFILE=1 set?")
|
||||
return
|
||||
last = summaries[-1] if summaries else {}
|
||||
print(f"=== {path} ===")
|
||||
if last:
|
||||
print(f"threshold : {last.get('threshold', 0)} bytes "
|
||||
f"(allocations below this are invisible by design)")
|
||||
print(f"sites seen : {last.get('sites', 0)}")
|
||||
print(f"attributed live: {last.get('live_bytes', 0)/1048576:.1f} MB "
|
||||
f"in {last.get('live_blocks', 0)} blocks")
|
||||
lost = last.get("site_table_full", 0) + last.get("pointer_table_full", 0)
|
||||
if lost:
|
||||
print(f"RECORDS LOST : {lost} — tables exhausted, treat totals as a floor")
|
||||
|
||||
growth = []
|
||||
for site, samples in series.items():
|
||||
live = [s["live_bytes"] for s in samples]
|
||||
slope = linear_slope(live)
|
||||
if slope <= 0 and live[-1] <= live[0]:
|
||||
continue
|
||||
growth.append((slope, live[-1] - live[0], live[-1], samples[-1]))
|
||||
growth.sort(key=lambda row: row[0], reverse=True)
|
||||
|
||||
print(f"\n--- sites whose retained bytes GROW (top {top}) ---")
|
||||
if not growth:
|
||||
print("none — no tracked site retains more over time.")
|
||||
for slope, delta, live, sample in growth[:top]:
|
||||
frames = " <- ".join(sample.get("frames", [])[:4])
|
||||
print(f" +{slope/1024:8.1f} KB/sample total +{delta/1048576:6.2f} MB "
|
||||
f"live {live/1048576:6.2f} MB")
|
||||
print(f" {frames}")
|
||||
|
||||
if census_path:
|
||||
rows = parse_census(census_path)
|
||||
if rows:
|
||||
priv = [r["priv_kb"] for r in rows if "priv_kb" in r]
|
||||
prof = [r["prof_live_kb"] for r in rows if "prof_live_kb" in r]
|
||||
print(f"\n--- pools vs attribution ({len(rows)} samples) ---")
|
||||
if priv:
|
||||
print(f" private committed : {priv[0]/1024:.1f} -> {priv[-1]/1024:.1f} MB "
|
||||
f"({linear_slope(priv):+.1f} KB/sample)")
|
||||
if prof and priv:
|
||||
unattributed = (priv[-1] - priv[0]) - (prof[-1] - prof[0])
|
||||
print(f" attributed growth : {(prof[-1]-prof[0])/1024:+.1f} MB")
|
||||
print(f" UNATTRIBUTED : {unattributed/1024:+.1f} MB "
|
||||
"<- below-threshold allocations, allocator metadata, or fragmentation")
|
||||
|
||||
|
||||
def diff(path_a, path_b, top):
|
||||
"""Same sites, two platforms: what does one retain that the other does not?"""
|
||||
_, series_a = load_profile(path_a)
|
||||
_, series_b = load_profile(path_b)
|
||||
rows = []
|
||||
for site, samples in series_a.items():
|
||||
a_live = samples[-1]["live_bytes"]
|
||||
b_live = series_b[site][-1]["live_bytes"] if site in series_b else 0
|
||||
if a_live - b_live > 0:
|
||||
rows.append((a_live - b_live, a_live, b_live, samples[-1]))
|
||||
rows.sort(key=lambda row: row[0], reverse=True)
|
||||
print(f"=== {path_a} minus {path_b} (top {top}) ===")
|
||||
for delta, a_live, b_live, sample in rows[:top]:
|
||||
print(f" +{delta/1048576:7.2f} MB A={a_live/1048576:.2f} MB B={b_live/1048576:.2f} MB")
|
||||
print(f" {' <- '.join(sample.get('frames', [])[:4])}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("profile")
|
||||
parser.add_argument("--census")
|
||||
parser.add_argument("--diff")
|
||||
parser.add_argument("--top", type=int, default=12)
|
||||
args = parser.parse_args()
|
||||
if args.diff:
|
||||
diff(args.profile, args.diff, args.top)
|
||||
else:
|
||||
report(args.profile, args.census, args.top)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+19
-1
@@ -10,6 +10,7 @@
|
||||
#include "platform.h"
|
||||
#include "log.h"
|
||||
#include "compat_thread.h"
|
||||
#include "mem_profile.h"
|
||||
|
||||
#include "foundation/constants.h"
|
||||
|
||||
@@ -858,7 +859,24 @@ void cbm_mem_census_log(const char *tag) {
|
||||
(void)snprintf(dc_err, sizeof(dc_err), "n/a");
|
||||
(void)snprintf(dc_mb, sizeof(dc_mb), "n/a");
|
||||
#endif
|
||||
cbm_log_info("mem.census", "at", tag ? tag : "?", "priv_kb", priv_kb, "crt_kb", crt_kb,
|
||||
/* Attribution alongside the pool totals: the census says which pool holds
|
||||
* the memory, the profile says which call sites put it there. Emitting
|
||||
* both from one place keeps the two views on the same sample. */
|
||||
cbm_mem_profile_totals_t profile;
|
||||
cbm_mem_profile_totals(&profile);
|
||||
char prof_live_kb[CBM_SZ_32];
|
||||
char prof_sites[CBM_SZ_32];
|
||||
char prof_lost[CBM_SZ_32];
|
||||
(void)snprintf(prof_live_kb, sizeof(prof_live_kb), "%zu", profile.live_bytes / CBM_SZ_1K);
|
||||
(void)snprintf(prof_sites, sizeof(prof_sites), "%zu", profile.sites);
|
||||
(void)snprintf(prof_lost, sizeof(prof_lost), "%zu",
|
||||
profile.site_table_full + profile.pointer_table_full);
|
||||
char profile_path[CBM_SZ_1K];
|
||||
if (cbm_safe_getenv("CBM_MEM_PROFILE_OUT", profile_path, sizeof(profile_path), NULL) != NULL) {
|
||||
(void)cbm_mem_profile_dump(profile_path, tag ? tag : "?");
|
||||
}
|
||||
cbm_log_info("mem.census", "at", tag ? tag : "?",
|
||||
"prof_live_kb", prof_live_kb, "prof_sites", prof_sites, "prof_lost", prof_lost, "priv_kb", priv_kb, "crt_kb", crt_kb,
|
||||
"crt_busy_kb", busy_kb, "mapped_kb", mapped_kb, "rss_kb", rss_kb, "biggest_kb",
|
||||
biggest_kb, "regions", regions, "big_regions", big_regions, "heaps", heaps,
|
||||
"decommit_calls", dc_calls, "decommit_fails", dc_fail, "decommit_err", dc_err,
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
* mimalloc is always safe.
|
||||
*/
|
||||
#include "foundation/mem.h"
|
||||
#include "foundation/mem_profile.h"
|
||||
|
||||
#include <mimalloc.h>
|
||||
#include <stddef.h>
|
||||
@@ -69,11 +70,15 @@ static inline bool mem_override_is_ours(const void *block) {
|
||||
}
|
||||
|
||||
void *__wrap_malloc(size_t size) {
|
||||
return mi_malloc(size);
|
||||
void *block = mi_malloc(size);
|
||||
cbm_mem_profile_alloc(block, size);
|
||||
return block;
|
||||
}
|
||||
|
||||
void *__wrap_calloc(size_t count, size_t size) {
|
||||
return mi_calloc(count, size);
|
||||
void *block = mi_calloc(count, size);
|
||||
cbm_mem_profile_alloc(block, count * size);
|
||||
return block;
|
||||
}
|
||||
|
||||
char *__wrap_strdup(const char *text) {
|
||||
@@ -95,6 +100,7 @@ void __wrap_free(void *block) {
|
||||
return;
|
||||
}
|
||||
if (mem_override_is_ours(block)) {
|
||||
cbm_mem_profile_free(block);
|
||||
mi_free(block);
|
||||
return;
|
||||
}
|
||||
@@ -120,7 +126,10 @@ void *__wrap_realloc(void *block, size_t size) {
|
||||
return mi_malloc(size);
|
||||
}
|
||||
if (mem_override_is_ours(block)) {
|
||||
return mi_realloc(block, size);
|
||||
cbm_mem_profile_free(block);
|
||||
void *grown = mi_realloc(block, size);
|
||||
cbm_mem_profile_alloc(grown, size);
|
||||
return grown;
|
||||
}
|
||||
/* A CRT block cannot be handed to mi_realloc, and the CRT's realloc must
|
||||
* not be asked to grow into mimalloc's space either. Copy across the
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
/*
|
||||
* mem_profile.c — see mem_profile.h.
|
||||
*
|
||||
* Two fixed tables, both open-addressed with linear probing:
|
||||
* sites[] stack hash -> {frames, live bytes, live blocks, totals}
|
||||
* pointers[] block address -> {site index, size}
|
||||
*
|
||||
* A single mutex guards both. That is heavy for a hot path and entirely
|
||||
* acceptable here: profiling is opt-in, and a diagnostic that is fast but
|
||||
* racy would produce numbers nobody can trust — which is the failure mode
|
||||
* this whole harness exists to correct.
|
||||
*/
|
||||
#include "mem_profile.h"
|
||||
|
||||
#include "compat_fs.h"
|
||||
#include "compat_thread.h"
|
||||
#include "foundation/constants.h"
|
||||
#include "platform.h"
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <execinfo.h>
|
||||
#endif
|
||||
|
||||
enum {
|
||||
PROFILE_FRAMES = 8, /* stack depth per site */
|
||||
PROFILE_SKIP_FRAMES = 2, /* this file's own frames */
|
||||
PROFILE_SITES = 4096, /* distinct call sites */
|
||||
PROFILE_POINTERS = 262144, /* tracked live blocks */
|
||||
PROFILE_DEFAULT_MIN = 32768,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uint64_t hash; /* 0 = empty slot */
|
||||
void *frames[PROFILE_FRAMES];
|
||||
int frame_count;
|
||||
size_t live_bytes;
|
||||
size_t live_blocks;
|
||||
size_t total_bytes;
|
||||
size_t total_blocks;
|
||||
size_t peak_live_bytes;
|
||||
} profile_site_t;
|
||||
|
||||
typedef struct {
|
||||
void *block; /* NULL = empty slot */
|
||||
size_t size;
|
||||
size_t site;
|
||||
} profile_pointer_t;
|
||||
|
||||
static cbm_mutex_t g_profile_mutex;
|
||||
static profile_site_t g_sites[PROFILE_SITES];
|
||||
static profile_pointer_t g_pointers[PROFILE_POINTERS];
|
||||
static cbm_mem_profile_totals_t g_totals;
|
||||
static atomic_int g_profile_enabled = -1;
|
||||
static size_t g_profile_min = PROFILE_DEFAULT_MIN;
|
||||
|
||||
size_t cbm_mem_profile_threshold(void) {
|
||||
(void)cbm_mem_profile_enabled();
|
||||
return g_profile_min;
|
||||
}
|
||||
|
||||
bool cbm_mem_profile_enabled(void) {
|
||||
int state = atomic_load_explicit(&g_profile_enabled, memory_order_acquire);
|
||||
if (state >= 0) {
|
||||
return state == 1;
|
||||
}
|
||||
char buf[CBM_SZ_32];
|
||||
bool on = cbm_safe_getenv("CBM_MEM_PROFILE", buf, sizeof(buf), NULL) != NULL && buf[0] == '1';
|
||||
if (on) {
|
||||
char min_buf[CBM_SZ_32];
|
||||
if (cbm_safe_getenv("CBM_MEM_PROFILE_MIN", min_buf, sizeof(min_buf), NULL) != NULL) {
|
||||
long parsed = strtol(min_buf, NULL, 10);
|
||||
if (parsed > 0) {
|
||||
g_profile_min = (size_t)parsed;
|
||||
}
|
||||
}
|
||||
cbm_mutex_init(&g_profile_mutex);
|
||||
}
|
||||
atomic_store_explicit(&g_profile_enabled, on ? 1 : 0, memory_order_release);
|
||||
return on;
|
||||
}
|
||||
|
||||
/* FNV-1a over the return addresses: cheap, and collisions merely merge two
|
||||
* sites rather than corrupting counts. */
|
||||
static uint64_t profile_hash(void *const *frames, int count) {
|
||||
uint64_t hash = 1469598103934665603ULL;
|
||||
for (int i = 0; i < count; i++) {
|
||||
uintptr_t value = (uintptr_t)frames[i];
|
||||
for (size_t byte = 0; byte < sizeof(value); byte++) {
|
||||
hash ^= (value >> (byte * 8)) & 0xFFU;
|
||||
hash *= 1099511628211ULL;
|
||||
}
|
||||
}
|
||||
return hash == 0 ? 1 : hash; /* 0 marks an empty slot */
|
||||
}
|
||||
|
||||
static int profile_capture(void **frames) {
|
||||
#ifdef _WIN32
|
||||
return (int)CaptureStackBackTrace(PROFILE_SKIP_FRAMES, PROFILE_FRAMES, frames, NULL);
|
||||
#else
|
||||
void *raw[PROFILE_FRAMES + PROFILE_SKIP_FRAMES];
|
||||
int got = backtrace(raw, PROFILE_FRAMES + PROFILE_SKIP_FRAMES);
|
||||
int count = got - PROFILE_SKIP_FRAMES;
|
||||
if (count <= 0) {
|
||||
return 0;
|
||||
}
|
||||
if (count > PROFILE_FRAMES) {
|
||||
count = PROFILE_FRAMES;
|
||||
}
|
||||
memcpy(frames, raw + PROFILE_SKIP_FRAMES, (size_t)count * sizeof(*frames));
|
||||
return count;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Returns SIZE_MAX when the table is full, so the caller can count the loss. */
|
||||
static size_t profile_intern_site_locked(void **frames, int count) {
|
||||
uint64_t hash = profile_hash(frames, count);
|
||||
size_t index = (size_t)(hash % PROFILE_SITES);
|
||||
for (size_t probe = 0; probe < PROFILE_SITES; probe++) {
|
||||
profile_site_t *site = &g_sites[index];
|
||||
if (site->hash == hash) {
|
||||
return index;
|
||||
}
|
||||
if (site->hash == 0) {
|
||||
site->hash = hash;
|
||||
site->frame_count = count;
|
||||
memcpy(site->frames, frames, (size_t)count * sizeof(*frames));
|
||||
g_totals.sites++;
|
||||
return index;
|
||||
}
|
||||
index = (index + 1) % PROFILE_SITES;
|
||||
}
|
||||
return SIZE_MAX;
|
||||
}
|
||||
|
||||
static size_t profile_pointer_slot(const void *block) {
|
||||
uintptr_t value = (uintptr_t)block;
|
||||
/* Block addresses are page-aligned at these sizes, so mix the high bits
|
||||
* down before masking or every entry lands in the same few buckets. */
|
||||
value ^= value >> 20;
|
||||
value *= 2654435761U;
|
||||
return (size_t)(value % PROFILE_POINTERS);
|
||||
}
|
||||
|
||||
void cbm_mem_profile_alloc(void *block, size_t size) {
|
||||
if (!block || !cbm_mem_profile_enabled() || size < g_profile_min) {
|
||||
return;
|
||||
}
|
||||
void *frames[PROFILE_FRAMES];
|
||||
int count = profile_capture(frames);
|
||||
if (count <= 0) {
|
||||
return;
|
||||
}
|
||||
cbm_mutex_lock(&g_profile_mutex);
|
||||
size_t site_index = profile_intern_site_locked(frames, count);
|
||||
if (site_index == SIZE_MAX) {
|
||||
g_totals.site_table_full++;
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
return;
|
||||
}
|
||||
size_t slot = profile_pointer_slot(block);
|
||||
bool stored = false;
|
||||
for (size_t probe = 0; probe < PROFILE_POINTERS; probe++) {
|
||||
if (g_pointers[slot].block == NULL) {
|
||||
g_pointers[slot].block = block;
|
||||
g_pointers[slot].size = size;
|
||||
g_pointers[slot].site = site_index;
|
||||
stored = true;
|
||||
break;
|
||||
}
|
||||
slot = (slot + 1) % PROFILE_POINTERS;
|
||||
}
|
||||
if (!stored) {
|
||||
g_totals.pointer_table_full++;
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
return;
|
||||
}
|
||||
profile_site_t *site = &g_sites[site_index];
|
||||
site->live_bytes += size;
|
||||
site->live_blocks++;
|
||||
site->total_bytes += size;
|
||||
site->total_blocks++;
|
||||
if (site->live_bytes > site->peak_live_bytes) {
|
||||
site->peak_live_bytes = site->live_bytes;
|
||||
}
|
||||
g_totals.live_bytes += size;
|
||||
g_totals.live_blocks++;
|
||||
g_totals.total_bytes += size;
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
}
|
||||
|
||||
void cbm_mem_profile_free(void *block) {
|
||||
if (!block || !cbm_mem_profile_enabled()) {
|
||||
return;
|
||||
}
|
||||
cbm_mutex_lock(&g_profile_mutex);
|
||||
size_t slot = profile_pointer_slot(block);
|
||||
for (size_t probe = 0; probe < PROFILE_POINTERS; probe++) {
|
||||
profile_pointer_t *entry = &g_pointers[slot];
|
||||
if (entry->block == block) {
|
||||
profile_site_t *site = &g_sites[entry->site];
|
||||
site->live_bytes -= entry->size;
|
||||
site->live_blocks--;
|
||||
g_totals.live_bytes -= entry->size;
|
||||
g_totals.live_blocks--;
|
||||
entry->block = NULL;
|
||||
entry->size = 0;
|
||||
entry->site = 0;
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
return;
|
||||
}
|
||||
if (entry->block == NULL) {
|
||||
break; /* linear probe ended: never tracked (below threshold) */
|
||||
}
|
||||
slot = (slot + 1) % PROFILE_POINTERS;
|
||||
}
|
||||
/* Expected for every sub-threshold allocation, so this is a scale check,
|
||||
* not an error: it should track the small-allocation rate, not grow with
|
||||
* retained bytes. */
|
||||
g_totals.untracked_frees++;
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
}
|
||||
|
||||
void cbm_mem_profile_totals(cbm_mem_profile_totals_t *out) {
|
||||
if (!out) {
|
||||
return;
|
||||
}
|
||||
memset(out, 0, sizeof(*out));
|
||||
if (!cbm_mem_profile_enabled()) {
|
||||
return;
|
||||
}
|
||||
cbm_mutex_lock(&g_profile_mutex);
|
||||
*out = g_totals;
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
}
|
||||
|
||||
bool cbm_mem_profile_dump(const char *path, const char *label) {
|
||||
if (!path || !cbm_mem_profile_enabled()) {
|
||||
return false;
|
||||
}
|
||||
FILE *out = cbm_fopen(path, "ab");
|
||||
if (!out) {
|
||||
return false;
|
||||
}
|
||||
cbm_mutex_lock(&g_profile_mutex);
|
||||
(void)fprintf(out,
|
||||
"{\"label\":\"%s\",\"threshold\":%zu,\"sites\":%zu,\"live_bytes\":%zu,"
|
||||
"\"live_blocks\":%zu,\"total_bytes\":%zu,\"untracked_frees\":%zu,"
|
||||
"\"site_table_full\":%zu,\"pointer_table_full\":%zu}\n",
|
||||
label ? label : "", g_profile_min, g_totals.sites, g_totals.live_bytes,
|
||||
g_totals.live_blocks, g_totals.total_bytes, g_totals.untracked_frees,
|
||||
g_totals.site_table_full, g_totals.pointer_table_full);
|
||||
for (size_t i = 0; i < PROFILE_SITES; i++) {
|
||||
profile_site_t *site = &g_sites[i];
|
||||
if (site->hash == 0 || site->total_blocks == 0) {
|
||||
continue;
|
||||
}
|
||||
(void)fprintf(out,
|
||||
"{\"label\":\"%s\",\"site\":%llu,\"live_bytes\":%zu,\"live_blocks\":%zu,"
|
||||
"\"total_bytes\":%zu,\"total_blocks\":%zu,\"peak_live\":%zu,\"frames\":[",
|
||||
label ? label : "", (unsigned long long)site->hash, site->live_bytes,
|
||||
site->live_blocks, site->total_bytes, site->total_blocks,
|
||||
site->peak_live_bytes);
|
||||
for (int f = 0; f < site->frame_count; f++) {
|
||||
(void)fprintf(out, "%s\"%p\"", f == 0 ? "" : ",", site->frames[f]);
|
||||
}
|
||||
(void)fprintf(out, "]}\n");
|
||||
}
|
||||
cbm_mutex_unlock(&g_profile_mutex);
|
||||
(void)fclose(out);
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* mem_profile.h — Which allocation sites retain memory?
|
||||
*
|
||||
* The census answers "which POOL holds the memory"; it cannot answer "which
|
||||
* allocation put it there". Without that, every hypothesis about #581 needed a
|
||||
* code change plus a soak to test, so the investigation could only ask one
|
||||
* yes/no question per eight minutes — and two mechanisms that looked confirmed
|
||||
* were later falsified. This records the missing half.
|
||||
*
|
||||
* Only allocations at or above a size threshold are tracked (default 32 KiB).
|
||||
* That is deliberate: arena fragmentation is driven by large transient runs,
|
||||
* and tracking every small allocation would cost more than the signal is worth.
|
||||
* The threshold is reported alongside the data so the blind spot is explicit.
|
||||
*
|
||||
* Off unless CBM_MEM_PROFILE=1. All tables are fixed-capacity and allocated
|
||||
* once; when a table fills, the overflow is COUNTED and reported rather than
|
||||
* silently dropped, because a profiler that quietly loses records produces
|
||||
* exactly the false confidence this is meant to end.
|
||||
*/
|
||||
#ifndef CBM_MEM_PROFILE_H
|
||||
#define CBM_MEM_PROFILE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/* Gate: CBM_MEM_PROFILE=1. Read once, cached. */
|
||||
bool cbm_mem_profile_enabled(void);
|
||||
|
||||
/* Minimum allocation size recorded, from CBM_MEM_PROFILE_MIN (bytes). */
|
||||
size_t cbm_mem_profile_threshold(void);
|
||||
|
||||
/* Record an allocation / release. Safe to call unconditionally: both return
|
||||
* immediately when profiling is off or the size is below the threshold. */
|
||||
void cbm_mem_profile_alloc(void *block, size_t size);
|
||||
void cbm_mem_profile_free(void *block);
|
||||
|
||||
typedef struct {
|
||||
size_t sites; /* distinct call sites seen */
|
||||
size_t live_bytes; /* currently retained, attributed */
|
||||
size_t live_blocks;
|
||||
size_t total_bytes; /* cumulative allocated at/above threshold */
|
||||
size_t untracked_frees; /* frees whose pointer was not in the table */
|
||||
size_t site_table_full; /* records lost: site table exhausted */
|
||||
size_t pointer_table_full; /* records lost: pointer table exhausted */
|
||||
} cbm_mem_profile_totals_t;
|
||||
|
||||
void cbm_mem_profile_totals(cbm_mem_profile_totals_t *out);
|
||||
|
||||
/* Append one JSON object per site to `path`, biggest live first, with the raw
|
||||
* return addresses of each site's stack. Symbolisation is deliberately left to
|
||||
* the offline analyser so the hot path never loads a symbol handler. */
|
||||
bool cbm_mem_profile_dump(const char *path, const char *label);
|
||||
|
||||
#endif /* CBM_MEM_PROFILE_H */
|
||||
Reference in new issue
Block a user