mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-06 15:49:59 +00:00
Martin Vogel
9c946d51a8
Three pools hold a whole project's proofs under fixed caps, and running out of one refuses the whole publication with no diagnostic: shared resolver 4M nodes, 64M work cbm_shared_resolver_build graph buffer 16M nodes, 4G work cbm_gbuf_new infra adapters 65536 nodes, 4M work cbm_shared_resolver_adapter_init proof_hash counts every hashed byte as work, so reaching a cap only means a large enough repository. django reached the shared cap at 78580 proof nodes, and because every proof hashes the project name, how soon a cap is reached also depends on the length of the checkout path. Measured, graph pool / shared pool work: redis (967K occurrences) 524M / 1.2M neovim (1.47M occurrences) 934M / 0.24M django (672K occurrences) 1.36G / 133M over the shared cap kubernetes (6.43M occurrences) 5.03G / 2.28G over both, and the adapter cap These pools hold one node per distinct project fact and each operation is linear in its own text, so the totals follow the input and are not guarding against a runaway. cbm_resolve_proof_pool_init_project sets no cap, so only the memory core can refuse. With this kubernetes publishes, 355464 nodes against main's 355619. Per-file pools keep their limits: they degrade the file's outcomes to LIMITED rather than stop publication. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
3258 lines
126 KiB
C
3258 lines
126 KiB
C
/*
|
|
* graph_buffer.c — In-memory graph buffer for pipeline indexing.
|
|
*
|
|
* Uses foundation hash tables for O(1) node lookup by QN and edge dedup.
|
|
* Uses dynamic arrays for ordered iteration and secondary indexes.
|
|
*
|
|
* Memory ownership: each node/edge is individually heap-allocated so that
|
|
* pointers stored in hash tables remain stable when the pointer-array grows.
|
|
* The buffer frees everything in cbm_gbuf_free().
|
|
*/
|
|
#include "foundation/constants.h"
|
|
|
|
enum {
|
|
GB_ERR = -1,
|
|
GB_COL_2 = 2,
|
|
GB_COL_3 = 3,
|
|
GB_COL_4 = 4,
|
|
GB_COL_5 = 5,
|
|
GB_COL_6 = 6,
|
|
GB_COL_7 = 7,
|
|
GB_URL_PATH_PREFIX = 12, /* strlen(""url_path":"") */
|
|
GB_MIN_FOR_DEDUP = 2, /* need at least 2 vectors to sort+dedup */
|
|
GB_DEDUP_LOOKAHEAD = 1, /* compare current with next element */
|
|
};
|
|
|
|
/* Sentinel for an edge property blob carrying no parseable "confidence": any
|
|
* blob that states one outranks a blob that does not. */
|
|
#define CBM_EDGE_CONF_ABSENT (-1.0)
|
|
#include "graph_buffer/graph_buffer.h"
|
|
#include <yyjson/yyjson.h> // url_path extraction must match json_extract semantics
|
|
#include "store/store.h"
|
|
#include "sqlite_writer.h"
|
|
#include "resolve/proof.h"
|
|
#include "resolve/policy.h"
|
|
#include "foundation/hash_table.h"
|
|
#include "foundation/compat.h"
|
|
#include "foundation/log.h"
|
|
#include "foundation/dyn_array.h"
|
|
#include "foundation/profile.h"
|
|
#include "foundation/mem.h"
|
|
#include "foundation/mem_core.h"
|
|
#include <sqlite3.h>
|
|
|
|
#include <errno.h>
|
|
#include <stdatomic.h>
|
|
#include <stdint.h> // int64_t
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h> // strdup
|
|
#include <time.h>
|
|
|
|
static inline void *intptr_to_ptr(intptr_t v) {
|
|
void *p;
|
|
memcpy(&p, &v, sizeof(p));
|
|
return p;
|
|
}
|
|
|
|
/* ── Internal types ──────────────────────────────────────────────── */
|
|
|
|
/* Edge key for dedup hash table — composite key as string "srcID:tgtID:type",
|
|
* plus ":local_name" for IMPORTS edges (#768). 256 bytes fit two int64s, the
|
|
* type and a ~200-char local_name verbatim; longer local_names are re-keyed
|
|
* with a hash of the full name in make_edge_key (never silently truncated). */
|
|
#define EDGE_KEY_BUF CBM_SZ_256
|
|
|
|
/* Per-type or per-key edge list stored in hash tables as values */
|
|
typedef struct {
|
|
uint64_t h0;
|
|
uint64_t h1;
|
|
cbm_gbuf_edge_t *edge; /* NULL = empty slot */
|
|
} edge_key_slot_t;
|
|
|
|
typedef struct {
|
|
edge_key_slot_t *slots;
|
|
size_t cap; /* power of two, 0 = not allocated */
|
|
size_t count;
|
|
} edge_key_map_t;
|
|
|
|
typedef CBM_DYN_ARRAY(const cbm_gbuf_edge_t *) edge_ptr_array_t;
|
|
|
|
/* Per-label or per-name node list */
|
|
typedef CBM_DYN_ARRAY(const cbm_gbuf_node_t *) node_ptr_array_t;
|
|
|
|
struct cbm_gbuf {
|
|
char *project;
|
|
char *root_path;
|
|
int64_t next_id;
|
|
_Atomic int64_t *shared_ids; /* NULL = use next_id, non-NULL = atomic source */
|
|
|
|
/* Node storage: array of pointers to individually heap-allocated nodes.
|
|
* This ensures pointers stored in hash tables remain valid when the
|
|
* pointer array reallocs (only the pointer array moves, not the nodes). */
|
|
CBM_DYN_ARRAY(cbm_gbuf_node_t *) nodes;
|
|
|
|
/* Primary index: QN → cbm_gbuf_node_t* */
|
|
CBMHashTable *node_by_qn;
|
|
/* Primary index: "id" string → cbm_gbuf_node_t* */
|
|
/* Dense id → node array (ids are sequential from alloc_next_id, shared
|
|
* with edges → holes where edges took ids). Replaces a hash table keyed
|
|
* on STRDUP'D DECIMAL STRINGS of the id — ~0.44 GB of buckets + key
|
|
* strings at kernel scale, plus a snprintf+strdup+hash on every one of
|
|
* the ~18 hot find_by_id call sites. */
|
|
cbm_gbuf_node_t **by_id;
|
|
/* Worker buffers (cbm_gbuf_new_worker) keep no by_id array: their ids
|
|
* come from the shared counter, so a dense array would span the whole
|
|
* global id space in every worker and double in lockstep. */
|
|
bool by_id_off;
|
|
int64_t by_id_cap;
|
|
|
|
/* Same-QN collision spans, written on export (cbm_gbuf_settle_variants).
|
|
* key: the surviving node's own QN string, value: gb_span_vec_t*. NULL
|
|
* until the first collision. */
|
|
CBMHashTable *variant_spans;
|
|
|
|
/* Secondary node indexes */
|
|
CBMHashTable *nodes_by_label; /* key: label, value: (node_ptr_array_t*) */
|
|
CBMHashTable *nodes_by_name; /* key: name, value: (node_ptr_array_t*) */
|
|
|
|
/* Edge storage: array of pointers to individually heap-allocated edges */
|
|
CBM_DYN_ARRAY(cbm_gbuf_edge_t *) edges;
|
|
|
|
cbm_resolve_record_t *resolutions;
|
|
size_t resolution_count;
|
|
size_t resolution_capacity;
|
|
bool resolution_failed;
|
|
CBMArena proof_arena;
|
|
cbm_resolve_proof_pool_t proof_pool;
|
|
cbm_resolve_proof_clone_session_t proof_session;
|
|
bool proof_session_active;
|
|
CBM_DYN_ARRAY(const cbm_resolve_import_snapshot_t *) import_snapshots;
|
|
CBMHashTable *imports_by_file;
|
|
CBM_DYN_ARRAY(cbm_resolve_file_t) resolver_files;
|
|
CBM_DYN_ARRAY(cbm_resolve_summary_t) resolver_summaries;
|
|
cbm_gbuf_captured_input_t *captured_inputs;
|
|
size_t captured_input_count, captured_input_capacity;
|
|
CBMHashTable *captured_input_paths;
|
|
bool captured_inputs_taken;
|
|
|
|
/* Edge dedup index: "srcID:tgtID:type" → cbm_gbuf_edge_t* */
|
|
edge_key_map_t edge_by_key; /* dedup: 128-bit key hash -> edge */
|
|
|
|
/* Edge secondary indexes: composite keys → edge_ptr_array_t */
|
|
CBMHashTable *edges_by_source_type; /* "srcID:type" → edge_ptr_array_t* */
|
|
CBMHashTable *edges_by_target_type; /* "tgtID:type" → edge_ptr_array_t* */
|
|
CBMHashTable *edges_by_type; /* "type" → edge_ptr_array_t* */
|
|
|
|
/* String intern pool for highly-repetitive fields (node label/file_path,
|
|
* edge type). Maps string content → owned canonical copy, collapsing
|
|
* O(nodes+edges) duplicate allocations to O(distinct). The pool owns the
|
|
* copies; interned pointers are stable for the buffer lifetime and are NOT
|
|
* freed by free_node_strings/free_edge_strings — only once in cbm_gbuf_free. */
|
|
CBMHashTable *intern_pool;
|
|
|
|
/* Vector storage for semantic embeddings (filled by pass_semantic_edges,
|
|
* consumed by cbm_write_db during dump). */
|
|
CBMDumpVector *dump_vectors;
|
|
int dump_vector_count;
|
|
int dump_vector_cap;
|
|
|
|
/* Token vector storage for enriched RI vectors (query-time lookup). */
|
|
CBMDumpTokenVec *dump_token_vecs;
|
|
int dump_token_vec_count;
|
|
int dump_token_vec_cap;
|
|
};
|
|
|
|
/* ── Helpers ─────────────────────────────────────────────────────── */
|
|
|
|
static char *heap_strdup(const char *s) {
|
|
return cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, s ? s : "{}");
|
|
}
|
|
|
|
/* Intern a repetitive string into the buffer's pool: identical content collapses
|
|
* to a single heap copy owned by the pool. NULL maps to "{}" (matches
|
|
* heap_strdup). The returned pointer is stable for the buffer's lifetime and
|
|
* must never be freed or mutated by callers. Returns NULL only on OOM. */
|
|
static const char *gb_intern(cbm_gbuf_t *gb, const char *s) {
|
|
const char *key = s ? s : "{}";
|
|
const char *found = cbm_ht_get(gb->intern_pool, key);
|
|
if (found) {
|
|
return found;
|
|
}
|
|
char *copy = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, key);
|
|
if (copy) {
|
|
cbm_ht_set(gb->intern_pool, copy, copy); /* key == value == owned copy */
|
|
}
|
|
return copy;
|
|
}
|
|
|
|
static bool captured_relative_path(const char *path) {
|
|
if (!path || !path[0] || path[0] == '/')
|
|
return false;
|
|
const char *segment = path;
|
|
for (const char *p = path;; p++) {
|
|
if (*p == '\\' || *p == ':' || (*p && (unsigned char)*p < 32) || *p == 127)
|
|
return false;
|
|
if (*p && *p != '/')
|
|
continue;
|
|
size_t length = (size_t)(p - segment);
|
|
if (!length || (length == 1 && segment[0] == '.') ||
|
|
(length == 2 && segment[0] == '.' && segment[1] == '.') || segment[length - 1] == '.' ||
|
|
segment[length - 1] == ' ')
|
|
return false;
|
|
if (!*p)
|
|
return true;
|
|
segment = p + 1;
|
|
}
|
|
}
|
|
|
|
static bool captured_digest(bool present, uint64_t length, const char *identity) {
|
|
if (!present)
|
|
return length == 0 && (!identity || !identity[0]);
|
|
if (length > INT64_MAX || !identity || strlen(identity) != CBM_SHA256_HEX_LEN)
|
|
return false;
|
|
for (size_t i = 0; i < CBM_SHA256_HEX_LEN; i++)
|
|
if (!((identity[i] >= '0' && identity[i] <= '9') ||
|
|
(identity[i] >= 'a' && identity[i] <= 'f')))
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
void cbm_gbuf_free_captured_inputs(cbm_gbuf_captured_input_t *inputs, size_t count) {
|
|
for (size_t i = 0; inputs && i < count; i++)
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, inputs[i].relative_path);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, inputs);
|
|
}
|
|
|
|
bool cbm_gbuf_capture_input(cbm_gbuf_t *gb, const char *relative_path, bool present,
|
|
uint64_t byte_length, const char *identity) {
|
|
if (!gb || gb->resolution_failed)
|
|
return false;
|
|
if (gb->captured_inputs_taken || !captured_relative_path(relative_path) ||
|
|
!captured_digest(present, byte_length, identity))
|
|
goto failed;
|
|
if (!gb->captured_input_paths)
|
|
gb->captured_input_paths = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 16);
|
|
if (!gb->captured_input_paths)
|
|
goto failed;
|
|
uintptr_t prior = (uintptr_t)cbm_ht_get(gb->captured_input_paths, relative_path);
|
|
if (prior) {
|
|
const cbm_gbuf_captured_input_t *input = &gb->captured_inputs[prior - 1];
|
|
if (input->present == present && input->byte_length == byte_length &&
|
|
!strcmp(input->content_identity, identity ? identity : ""))
|
|
return true;
|
|
goto failed;
|
|
}
|
|
if (gb->captured_input_count == gb->captured_input_capacity) {
|
|
size_t capacity = gb->captured_input_capacity ? gb->captured_input_capacity * 2 : 16;
|
|
if (capacity < gb->captured_input_capacity ||
|
|
capacity > SIZE_MAX / sizeof(*gb->captured_inputs))
|
|
goto failed;
|
|
cbm_gbuf_captured_input_t *items =
|
|
cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->captured_inputs, capacity * sizeof(*items));
|
|
if (!items)
|
|
goto failed;
|
|
gb->captured_inputs = items;
|
|
gb->captured_input_capacity = capacity;
|
|
}
|
|
char *path = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, relative_path);
|
|
if (!path)
|
|
goto failed;
|
|
size_t index = gb->captured_input_count;
|
|
cbm_gbuf_captured_input_t *input = &gb->captured_inputs[index];
|
|
*input = (cbm_gbuf_captured_input_t){
|
|
.relative_path = path, .present = present, .byte_length = byte_length};
|
|
if (present)
|
|
memcpy(input->content_identity, identity, sizeof(input->content_identity));
|
|
gb->captured_input_count++;
|
|
cbm_ht_set(gb->captured_input_paths, path, (void *)(uintptr_t)(index + 1));
|
|
if ((uintptr_t)cbm_ht_get(gb->captured_input_paths, path) != index + 1)
|
|
goto failed;
|
|
return true;
|
|
failed:
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
|
|
bool cbm_gbuf_take_captured_inputs(cbm_gbuf_t *gb, cbm_gbuf_captured_input_t **inputs,
|
|
size_t *count) {
|
|
if (inputs)
|
|
*inputs = NULL;
|
|
if (count)
|
|
*count = 0;
|
|
if (!gb || !inputs || !count || gb->resolution_failed || gb->captured_inputs_taken)
|
|
return false;
|
|
*inputs = gb->captured_inputs;
|
|
*count = gb->captured_input_count;
|
|
gb->captured_inputs = NULL;
|
|
gb->captured_input_count = 0;
|
|
gb->captured_input_capacity = 0;
|
|
cbm_ht_free(gb->captured_input_paths);
|
|
gb->captured_input_paths = NULL;
|
|
gb->captured_inputs_taken = true;
|
|
return true;
|
|
}
|
|
|
|
bool cbm_gbuf_resolution_begin(cbm_gbuf_t *gb) {
|
|
if (!gb || gb->resolution_failed || gb->proof_session_active)
|
|
return false;
|
|
gb->proof_session_active = cbm_resolve_proof_clone_begin(&gb->proof_session, &gb->proof_pool);
|
|
if (!gb->proof_session_active)
|
|
gb->resolution_failed = true;
|
|
return gb->proof_session_active;
|
|
}
|
|
|
|
void cbm_gbuf_resolution_end(cbm_gbuf_t *gb) {
|
|
if (gb && gb->proof_session_active) {
|
|
cbm_resolve_proof_clone_end(&gb->proof_session);
|
|
gb->proof_session_active = false;
|
|
}
|
|
}
|
|
|
|
static const cbm_resolve_proof_t *gb_copy_proof(cbm_gbuf_t *gb, const cbm_resolve_proof_t *proof) {
|
|
if (!proof)
|
|
return NULL;
|
|
return gb->proof_session_active ? cbm_resolve_proof_clone_root(&gb->proof_session, proof)
|
|
: cbm_resolve_proof_clone(&gb->proof_pool, proof);
|
|
}
|
|
|
|
void cbm_gbuf_forget_imports(cbm_gbuf_t *gb, const char *file) {
|
|
if (gb && gb->imports_by_file && file)
|
|
(void)cbm_ht_delete(gb->imports_by_file, file);
|
|
}
|
|
|
|
const cbm_resolve_import_snapshot_t *cbm_gbuf_imports(const cbm_gbuf_t *gb, const char *file) {
|
|
return gb && !gb->resolution_failed && gb->imports_by_file && file
|
|
? cbm_ht_get(gb->imports_by_file, file)
|
|
: NULL;
|
|
}
|
|
|
|
static const char *gb_import_text(cbm_gbuf_t *gb, const char *text) {
|
|
const char *copy = text ? gb_intern(gb, text) : NULL;
|
|
if (text && !copy)
|
|
gb->resolution_failed = true;
|
|
return copy;
|
|
}
|
|
|
|
bool cbm_gbuf_capture_imports(cbm_gbuf_t *gb, const cbm_resolve_import_snapshot_t *snapshot) {
|
|
if (!gb || !snapshot || !snapshot->source.file || gb->resolution_failed)
|
|
return false;
|
|
cbm_gbuf_forget_imports(gb, snapshot->source.file);
|
|
if ((snapshot->count && !snapshot->items) ||
|
|
snapshot->count > SIZE_MAX / sizeof(cbm_resolve_import_fact_t)) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
if (!gb->imports_by_file)
|
|
gb->imports_by_file = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 16);
|
|
cbm_resolve_import_snapshot_t *copy = cbm_arena_alloc(&gb->proof_arena, sizeof(*copy));
|
|
cbm_resolve_import_fact_t *items =
|
|
snapshot->count ? cbm_arena_alloc(&gb->proof_arena, snapshot->count * sizeof(*items))
|
|
: NULL;
|
|
if (!gb->imports_by_file || !copy || (snapshot->count && !items)) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
*copy = *snapshot;
|
|
copy->items = items;
|
|
#define IMPORT_SOURCE(field) copy->source.field = gb_import_text(gb, snapshot->source.field)
|
|
IMPORT_SOURCE(project);
|
|
IMPORT_SOURCE(file);
|
|
IMPORT_SOURCE(scope);
|
|
IMPORT_SOURCE(module);
|
|
IMPORT_SOURCE(content_identity);
|
|
IMPORT_SOURCE(configuration_identity);
|
|
#undef IMPORT_SOURCE
|
|
for (size_t i = 0; i < snapshot->count; i++) {
|
|
const cbm_resolve_import_fact_t *fact = &snapshot->items[i];
|
|
items[i] = *fact;
|
|
items[i].local_name = gb_import_text(gb, fact->local_name);
|
|
items[i].module_path = gb_import_text(gb, fact->module_path);
|
|
items[i].member = gb_import_text(gb, fact->member);
|
|
items[i].dependency.target = gb_import_text(gb, fact->dependency.target);
|
|
items[i].dependency.reason = gb_import_text(gb, fact->dependency.reason);
|
|
items[i].dependency.proof = gb_copy_proof(gb, fact->dependency.proof);
|
|
if (fact->dependency.proof && !items[i].dependency.proof)
|
|
gb->resolution_failed = true;
|
|
items[i].binding.target = gb_import_text(gb, fact->binding.target);
|
|
items[i].binding.reason = gb_import_text(gb, fact->binding.reason);
|
|
items[i].binding.proof = gb_copy_proof(gb, fact->binding.proof);
|
|
if (fact->binding.proof && !items[i].binding.proof)
|
|
gb->resolution_failed = true;
|
|
}
|
|
if (gb->resolution_failed)
|
|
return false;
|
|
int before = gb->import_snapshots.count;
|
|
cbm_da_push(&gb->import_snapshots, copy);
|
|
if (gb->import_snapshots.count != before + 1) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
cbm_ht_set(gb->imports_by_file, copy->source.file, copy);
|
|
gb->resolution_failed |= cbm_ht_get(gb->imports_by_file, copy->source.file) != copy;
|
|
return !gb->resolution_failed;
|
|
}
|
|
|
|
bool cbm_gbuf_record_resolution(cbm_gbuf_t *gb, const cbm_resolve_record_t *record) {
|
|
if (!gb || !record || gb->resolution_failed)
|
|
return false;
|
|
if (gb->resolution_count == gb->resolution_capacity) {
|
|
size_t next = gb->resolution_capacity ? gb->resolution_capacity * 2 : 64;
|
|
if (next < gb->resolution_capacity || next > SIZE_MAX / sizeof(*gb->resolutions)) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
cbm_resolve_record_t *rows =
|
|
cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->resolutions, next * sizeof(*rows));
|
|
if (!rows) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
gb->resolutions = rows;
|
|
gb->resolution_capacity = next;
|
|
}
|
|
cbm_resolve_record_t row = *record;
|
|
if (!row.proof_borrowed && row.result.proof) {
|
|
row.result.proof = gb_copy_proof(gb, row.result.proof);
|
|
if (!row.result.proof) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
}
|
|
if (row.binding_proof) {
|
|
row.binding_proof = gb_copy_proof(gb, row.binding_proof);
|
|
if (!row.binding_proof) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
}
|
|
row.content_identity = gb_intern(gb, record->content_identity ? record->content_identity : "");
|
|
row.configuration_identity =
|
|
gb_intern(gb, record->configuration_identity ? record->configuration_identity : "");
|
|
row.file_path = gb_intern(gb, record->file_path ? record->file_path : "");
|
|
row.scope = gb_intern(gb, record->scope ? record->scope : "");
|
|
row.relation = gb_intern(gb, record->relation ? record->relation : "");
|
|
row.spelling = gb_intern(gb, record->spelling ? record->spelling : "");
|
|
row.properties_json = gb_intern(gb, record->properties_json ? record->properties_json : "{}");
|
|
row.result.target = gb_intern(gb, record->result.target ? record->result.target : "");
|
|
row.result.reason = gb_intern(gb, record->result.reason ? record->result.reason : "");
|
|
if (!row.file_path || !row.scope || !row.relation || !row.spelling || !row.result.target ||
|
|
!row.result.reason || !row.properties_json || !row.content_identity ||
|
|
!row.configuration_identity) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
gb->resolutions[gb->resolution_count++] = row;
|
|
return true;
|
|
}
|
|
|
|
const cbm_resolve_record_t *cbm_gbuf_resolutions(const cbm_gbuf_t *gb, size_t *count) {
|
|
if (count)
|
|
*count = gb ? gb->resolution_count : 0;
|
|
return gb ? gb->resolutions : NULL;
|
|
}
|
|
|
|
bool cbm_gbuf_resolutions_complete(const cbm_gbuf_t *gb) {
|
|
return gb && !gb->resolution_failed;
|
|
}
|
|
|
|
void cbm_gbuf_fail_resolutions(cbm_gbuf_t *gb) {
|
|
if (gb)
|
|
gb->resolution_failed = true;
|
|
}
|
|
|
|
uint64_t cbm_gbuf_resolution_proof_work(const cbm_gbuf_t *gb) {
|
|
return gb ? gb->proof_pool.work : 0;
|
|
}
|
|
|
|
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
|
|
void cbm_gbuf_test_add_resolution_proof_work(cbm_gbuf_t *gb, uint64_t work) {
|
|
if (gb)
|
|
gb->proof_pool.work += work;
|
|
}
|
|
#endif
|
|
|
|
bool cbm_gbuf_resolutions_own_shared(cbm_gbuf_t *gb) {
|
|
if (!cbm_gbuf_resolution_begin(gb))
|
|
return false;
|
|
bool ok = true;
|
|
for (size_t i = 0; ok && i < gb->resolution_count; i++) {
|
|
cbm_resolve_record_t *row = &gb->resolutions[i];
|
|
if (!row->proof_borrowed)
|
|
continue;
|
|
row->result.proof = gb_copy_proof(gb, row->result.proof);
|
|
row->proof_borrowed = false;
|
|
ok = row->result.proof != NULL;
|
|
}
|
|
for (int i = 0; ok && i < gb->resolver_summaries.count; i++) {
|
|
cbm_resolve_summary_t *row = &gb->resolver_summaries.items[i];
|
|
if (!row->proof_borrowed)
|
|
continue;
|
|
row->result.proof = gb_copy_proof(gb, row->result.proof);
|
|
row->proof_borrowed = false;
|
|
ok = row->result.proof != NULL;
|
|
}
|
|
cbm_gbuf_resolution_end(gb);
|
|
/* The long shared ancestors are now interned once. Each occurrence adds
|
|
* one constant-size join with its local source binding validation. */
|
|
for (size_t i = 0; ok && i < gb->resolution_count; i++) {
|
|
cbm_resolve_record_t *row = &gb->resolutions[i];
|
|
if (!row->result.proof)
|
|
continue;
|
|
/* Replaying the coordinator does not add another envelope or change
|
|
* an already owned, admitted occurrence in this project. */
|
|
if (cbm_resolve_record_admit(row) && row->result.proof->value.source.project &&
|
|
!strcmp(row->result.proof->value.source.project, gb->project))
|
|
continue;
|
|
const cbm_resolve_proof_t *premises[] = {row->result.proof, row->binding_proof};
|
|
cbm_resolve_proof_source_t source = {.project = gb->project,
|
|
.file = row->file_path,
|
|
.scope = row->scope,
|
|
.content_identity = row->content_identity,
|
|
.configuration_identity = row->configuration_identity,
|
|
.source_origin = row->source_origin};
|
|
cbm_resolve_proof_key_t key = {.lookup_domain = row->scope,
|
|
.lookup_key = row->spelling,
|
|
.start_byte = row->start_byte,
|
|
.end_byte = row->end_byte,
|
|
.origin = row->source_origin,
|
|
.binding_identity = row->result.target,
|
|
.detail = row->relation};
|
|
cbm_resolve_result_t outcome = row->result;
|
|
outcome.proof = NULL;
|
|
uint32_t supported = premises[0]->value.outcome.facts;
|
|
if (row->binding_proof)
|
|
supported |= row->binding_proof->value.outcome.facts;
|
|
uint32_t required = outcome.facts;
|
|
const cbm_resolve_proof_view_t *binding = cbm_resolve_proof_view(row->binding_proof);
|
|
/* An incomplete local root must retain its own UNKNOWN/LIMITED/FAILED
|
|
* status through derive. A complete root must connect this exact site
|
|
* and role to the selected semantic target. Without one, the hybrid
|
|
* producer's root itself must provide that complete attestation. */
|
|
bool site_matches =
|
|
binding ? binding->outcome.analysis != CBM_RESOLVE_ANALYSIS_COMPLETE ||
|
|
cbm_resolve_record_site_proof(row, row->binding_proof, gb->project)
|
|
: cbm_resolve_record_site_proof(row, row->result.proof, gb->project);
|
|
/* Validate the semantic root before replacing it with the occurrence
|
|
* envelope. Local binding evidence may supply scope/import facts, but
|
|
* cannot change the root's selected target, domain or uniqueness. */
|
|
if (row->emitted &&
|
|
(!cbm_resolve_record_candidate(row) ||
|
|
premises[0]->value.outcome.candidate_count != outcome.candidate_count)) {
|
|
outcome.analysis = CBM_RESOLVE_ANALYSIS_UNSUPPORTED;
|
|
outcome.facts = 0;
|
|
outcome.reason = "proof_outcome_conflict";
|
|
} else if (row->emitted && !site_matches) {
|
|
outcome.analysis = CBM_RESOLVE_ANALYSIS_UNSUPPORTED;
|
|
outcome.facts = 0;
|
|
outcome.reason = "proof_site_conflict";
|
|
} else if (row->emitted && (supported & required) != required) {
|
|
outcome.analysis = CBM_RESOLVE_ANALYSIS_UNSUPPORTED;
|
|
outcome.facts = 0;
|
|
outcome.reason = "proof_authority_incomplete";
|
|
}
|
|
row->result.proof =
|
|
cbm_resolve_proof_derive(&gb->proof_pool, CBM_PROOF_OCCURRENCE_BINDING, &source, &key,
|
|
&outcome, premises, row->binding_proof ? 2 : 1, true);
|
|
ok = row->result.proof != NULL;
|
|
if (ok && row->emitted && !cbm_resolve_record_admit(row)) {
|
|
row->result.analysis = row->result.proof->value.outcome.analysis;
|
|
row->result.facts = 0;
|
|
row->result.reason = "proof_inputs_incomplete";
|
|
row->emitted = false;
|
|
}
|
|
}
|
|
gb->resolution_failed |= !ok;
|
|
return ok;
|
|
}
|
|
|
|
bool cbm_gbuf_record_resolver_file(cbm_gbuf_t *gb, const cbm_resolve_file_t *file) {
|
|
if (!gb || !file || gb->resolution_failed)
|
|
return false;
|
|
cbm_resolve_file_t row = *file;
|
|
#define INTERN_FILE(field) \
|
|
row.field = gb_intern(gb, file->field ? file->field : ""); \
|
|
if (!row.field) { \
|
|
gb->resolution_failed = true; \
|
|
return false; \
|
|
}
|
|
INTERN_FILE(file_path)
|
|
INTERN_FILE(module)
|
|
INTERN_FILE(raw_content_identity)
|
|
INTERN_FILE(preprocessed_content_identity)
|
|
INTERN_FILE(configuration_identity)
|
|
INTERN_FILE(extracted_summary)
|
|
#undef INTERN_FILE
|
|
int before = gb->resolver_files.count;
|
|
cbm_da_push(&gb->resolver_files, row);
|
|
gb->resolution_failed |= gb->resolver_files.count != before + 1;
|
|
return !gb->resolution_failed;
|
|
}
|
|
|
|
const cbm_resolve_file_t *cbm_gbuf_resolver_files(const cbm_gbuf_t *gb, size_t *count) {
|
|
if (count)
|
|
*count = gb ? (size_t)gb->resolver_files.count : 0;
|
|
return gb ? gb->resolver_files.items : NULL;
|
|
}
|
|
|
|
bool cbm_gbuf_record_resolver_summary(cbm_gbuf_t *gb, const cbm_resolve_summary_t *summary) {
|
|
if (!gb || !summary || gb->resolution_failed)
|
|
return false;
|
|
cbm_resolve_summary_t row = *summary;
|
|
row.file_path = gb_intern(gb, row.file_path ? row.file_path : "");
|
|
row.key = gb_intern(gb, row.key ? row.key : "");
|
|
row.payload_json = gb_intern(gb, row.payload_json ? row.payload_json : "{}");
|
|
row.result.target = gb_intern(gb, row.result.target ? row.result.target : "");
|
|
row.result.reason = gb_intern(gb, row.result.reason ? row.result.reason : "");
|
|
if (row.result.proof && !row.proof_borrowed)
|
|
row.result.proof = gb_copy_proof(gb, row.result.proof);
|
|
if (!row.file_path || !row.key || !row.payload_json || !row.result.target ||
|
|
!row.result.reason || (summary->result.proof && !row.result.proof)) {
|
|
gb->resolution_failed = true;
|
|
return false;
|
|
}
|
|
int before = gb->resolver_summaries.count;
|
|
cbm_da_push(&gb->resolver_summaries, row);
|
|
gb->resolution_failed |= gb->resolver_summaries.count != before + 1;
|
|
return !gb->resolution_failed;
|
|
}
|
|
|
|
const cbm_resolve_summary_t *cbm_gbuf_resolver_summaries(const cbm_gbuf_t *gb, size_t *count) {
|
|
if (count)
|
|
*count = gb ? (size_t)gb->resolver_summaries.count : 0;
|
|
return gb ? gb->resolver_summaries.items : NULL;
|
|
}
|
|
|
|
static void make_id_key(char *buf, size_t bufsz, int64_t id) {
|
|
snprintf(buf, bufsz, "%lld", (long long)id);
|
|
}
|
|
|
|
/* FNV-1a 64-bit over a byte slice — for re-keying oversized local_names. */
|
|
static uint64_t fnv1a64(const char *s, size_t len) {
|
|
uint64_t h = 14695981039346656037ULL;
|
|
for (size_t i = 0; i < len; i++) {
|
|
h ^= (uint8_t)s[i];
|
|
h *= 1099511628211ULL;
|
|
}
|
|
return h;
|
|
}
|
|
|
|
/* IMPORTS edges carry exactly one imported symbol's local_name (#768): two
|
|
* named imports from the same specifier resolve to the same (source,
|
|
* target) pair but are distinct symbols. Key on local_name too so the
|
|
* second import doesn't dedup-collide with and overwrite the first —
|
|
* every pass that walks IMPORTS edges (pass_calls.c, pass_usages.c,
|
|
* pass_semantic.c, pass_lsp_cross.c) expects one local_name per edge, so
|
|
* losing an edge here silently breaks cross-file call resolution for
|
|
* whichever symbol got dropped, not just "who imports X" queries. Other
|
|
* edge types keep the plain (source,target,type) key: collapsing repeat
|
|
* edges of the same type between the same two nodes (e.g. multiple call
|
|
* sites) into one is the existing, intended dedup behavior there.
|
|
*
|
|
* A local_name too long for the key buffer is re-keyed with an FNV-1a hash
|
|
* of the FULL name instead of being truncated — a truncated key would
|
|
* collide two long names sharing a prefix and silently drop an edge again.
|
|
* The hash key is prefixed with byte 0x01, which cannot appear in the raw
|
|
* JSON slice (control characters must be \u-escaped in JSON), so hash keys
|
|
* can never collide with verbatim keys. */
|
|
static void make_edge_key(char *buf, size_t bufsz, int64_t src, int64_t tgt, const char *type,
|
|
const char *properties_json) {
|
|
if (properties_json && strcmp(type, "IMPORTS") == 0) {
|
|
static const char local_name_key[] = "\"local_name\":\"";
|
|
const char *ln = strstr(properties_json, local_name_key);
|
|
if (ln) {
|
|
ln += sizeof(local_name_key) - 1;
|
|
const char *end = strchr(ln, '"');
|
|
size_t ln_len = end ? (size_t)(end - ln) : strlen(ln);
|
|
int n = snprintf(buf, bufsz, "%lld:%lld:%s:%.*s", (long long)src, (long long)tgt, type,
|
|
(int)ln_len, ln);
|
|
if (n < 0 || (size_t)n >= bufsz) {
|
|
snprintf(buf, bufsz, "%lld:%lld:%s:\x01%016llx", (long long)src, (long long)tgt,
|
|
type, (unsigned long long)fnv1a64(ln, ln_len));
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
snprintf(buf, bufsz, "%lld:%lld:%s", (long long)src, (long long)tgt, type);
|
|
}
|
|
|
|
static void make_src_type_key(char *buf, size_t bufsz, int64_t src, const char *type) {
|
|
snprintf(buf, bufsz, "%lld:%s", (long long)src, type);
|
|
}
|
|
|
|
/* ── Edge dedup index: 128-bit key hash -> edge, open addressing ──────
|
|
*
|
|
* The dedup key is "src:tgt:type:props" (make_edge_key). Keeping it as a
|
|
* string per edge cost the kernel 15.8M heap strings (~1.2 GB live, and the
|
|
* same again transiently in worker buffers during merge). Two independent
|
|
* 64-bit FNV-1a hashes of the composed key take 16 bytes; a false match needs
|
|
* both to collide, ~n^2 / 2^129 for n edges -- below any other failure mode of
|
|
* the index. Linear probing, backward-shift deletion (no tombstones), load
|
|
* kept under 1/2. Values are the edge pointers (stable heap records). */
|
|
|
|
static void edge_key_hash(const char *key, uint64_t *h0, uint64_t *h1) {
|
|
uint64_t a = 1469598103934665603ULL;
|
|
uint64_t b = 0x9E3779B97F4A7C15ULL;
|
|
for (const unsigned char *p = (const unsigned char *)key; *p; p++) {
|
|
a ^= *p;
|
|
a *= 1099511628211ULL;
|
|
b = (b ^ *p) * 0xFF51AFD7ED558CCDULL;
|
|
b ^= b >> 29;
|
|
}
|
|
*h0 = a;
|
|
*h1 = b ^ (a >> 17);
|
|
}
|
|
|
|
static void edge_key_map_free(edge_key_map_t *m) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, m->slots);
|
|
m->slots = NULL;
|
|
m->cap = 0;
|
|
m->count = 0;
|
|
}
|
|
|
|
static edge_key_slot_t *edge_key_map_find(const edge_key_map_t *m, uint64_t h0, uint64_t h1) {
|
|
if (!m->slots) {
|
|
return NULL;
|
|
}
|
|
size_t mask = m->cap - SKIP_ONE;
|
|
for (size_t i = (size_t)h0 & mask;; i = (i + SKIP_ONE) & mask) {
|
|
edge_key_slot_t *s = &m->slots[i];
|
|
if (!s->edge) {
|
|
return NULL;
|
|
}
|
|
if (s->h0 == h0 && s->h1 == h1) {
|
|
return s;
|
|
}
|
|
}
|
|
}
|
|
|
|
static bool edge_key_map_grow(edge_key_map_t *m) {
|
|
size_t new_cap = m->cap ? m->cap * PAIR_LEN : CBM_SZ_1K;
|
|
edge_key_slot_t *slots = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, new_cap * sizeof(*slots));
|
|
if (!slots) {
|
|
return false;
|
|
}
|
|
size_t mask = new_cap - SKIP_ONE;
|
|
for (size_t i = 0; i < m->cap; i++) {
|
|
edge_key_slot_t *s = &m->slots[i];
|
|
if (!s->edge) {
|
|
continue;
|
|
}
|
|
size_t j = (size_t)s->h0 & mask;
|
|
while (slots[j].edge) {
|
|
j = (j + SKIP_ONE) & mask;
|
|
}
|
|
slots[j] = *s;
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, m->slots);
|
|
m->slots = slots;
|
|
m->cap = new_cap;
|
|
return true;
|
|
}
|
|
|
|
static bool edge_key_map_set(edge_key_map_t *m, uint64_t h0, uint64_t h1, cbm_gbuf_edge_t *edge) {
|
|
if ((m->count + SKIP_ONE) * PAIR_LEN > m->cap && !edge_key_map_grow(m)) {
|
|
return false;
|
|
}
|
|
size_t mask = m->cap - SKIP_ONE;
|
|
size_t i = (size_t)h0 & mask;
|
|
while (m->slots[i].edge) {
|
|
if (m->slots[i].h0 == h0 && m->slots[i].h1 == h1) {
|
|
m->slots[i].edge = edge;
|
|
return true;
|
|
}
|
|
i = (i + SKIP_ONE) & mask;
|
|
}
|
|
m->slots[i].h0 = h0;
|
|
m->slots[i].h1 = h1;
|
|
m->slots[i].edge = edge;
|
|
m->count++;
|
|
return true;
|
|
}
|
|
|
|
static void edge_key_map_delete(edge_key_map_t *m, uint64_t h0, uint64_t h1) {
|
|
edge_key_slot_t *s = edge_key_map_find(m, h0, h1);
|
|
if (!s) {
|
|
return;
|
|
}
|
|
size_t mask = m->cap - SKIP_ONE;
|
|
size_t i = (size_t)(s - m->slots);
|
|
m->slots[i].edge = NULL;
|
|
m->count--;
|
|
/* Backward shift: pull later entries of the same probe run into the hole. */
|
|
for (size_t j = (i + SKIP_ONE) & mask; m->slots[j].edge; j = (j + SKIP_ONE) & mask) {
|
|
size_t home = (size_t)m->slots[j].h0 & mask;
|
|
/* entry at j may move to i if its home is not in (i, j] cyclically */
|
|
bool movable = (i <= j) ? (home <= i || home > j) : (home <= i && home > j);
|
|
if (movable) {
|
|
m->slots[i] = m->slots[j];
|
|
m->slots[j].edge = NULL;
|
|
i = j;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Get or create a node_ptr_array_t in a hash table */
|
|
static void node_array_push(node_ptr_array_t *arr, const cbm_gbuf_node_t *node);
|
|
|
|
static node_ptr_array_t *get_or_create_node_array(CBMHashTable *ht, const char *key) {
|
|
if (!ht) {
|
|
return NULL; /* worker buffer: no secondary indexes (cbm_gbuf_new_worker) */
|
|
}
|
|
node_ptr_array_t *arr = cbm_ht_get(ht, key);
|
|
if (!arr) {
|
|
arr = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(node_ptr_array_t));
|
|
cbm_ht_set(ht, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), arr);
|
|
}
|
|
return arr;
|
|
}
|
|
|
|
/* Get or create an edge_ptr_array_t in a hash table */
|
|
static edge_ptr_array_t *get_or_create_edge_array(CBMHashTable *ht, const char *key) {
|
|
if (!ht) {
|
|
return NULL;
|
|
}
|
|
edge_ptr_array_t *arr = cbm_ht_get(ht, key);
|
|
if (!arr) {
|
|
arr = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(edge_ptr_array_t));
|
|
cbm_ht_set(ht, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), arr);
|
|
}
|
|
return arr;
|
|
}
|
|
|
|
/* Free a node_ptr_array_t (callback for hash table iteration) */
|
|
static void free_node_array(const char *key, void *value, void *ud) {
|
|
(void)ud;
|
|
node_ptr_array_t *arr = value;
|
|
if (arr) {
|
|
cbm_da_free(arr);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, arr);
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
|
|
}
|
|
|
|
/* Free an edge_ptr_array_t (callback) */
|
|
static void free_edge_array(const char *key, void *value, void *ud) {
|
|
(void)ud;
|
|
edge_ptr_array_t *arr = value;
|
|
if (arr) {
|
|
cbm_da_free(arr);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, arr);
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
|
|
}
|
|
|
|
/* Free index keys only (deleted_set id keys). */
|
|
static void free_index_key(const char *key, void *value, void *ud) {
|
|
(void)value;
|
|
(void)ud;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
|
|
}
|
|
|
|
/* Free one intern-pool entry: key == value == the single owned string copy
|
|
* that nodes/edges borrowed for label/file_path/type. */
|
|
static void free_intern_entry(const char *key, void *value, void *ud) {
|
|
(void)value;
|
|
(void)ud;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, (void *)key);
|
|
}
|
|
|
|
/* Free a single node's owned strings. label and file_path are interned
|
|
* (pool-owned) — NOT freed here; the pool frees them once in cbm_gbuf_free. */
|
|
static void free_node_strings(cbm_gbuf_node_t *n) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->name);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->qualified_name);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->properties_json);
|
|
}
|
|
|
|
/* An edge owns no strings: type AND properties_json are interned (pool-owned;
|
|
* the pool frees each once in cbm_gbuf_free). Edge properties repeat massively
|
|
* -- on the Go corpus 1.8M edges carry 343k distinct property strings and 854k
|
|
* are exactly "{}" -- and an edge's properties are never mutated in place. */
|
|
static void free_edge_strings(cbm_gbuf_edge_t *e) {
|
|
(void)e;
|
|
}
|
|
|
|
/* Allocate the next buffer-local or shared-atomic ID. */
|
|
static int64_t alloc_next_id(cbm_gbuf_t *gb) {
|
|
if (gb->shared_ids) {
|
|
return atomic_fetch_add_explicit(gb->shared_ids, SKIP_ONE, memory_order_relaxed);
|
|
}
|
|
return gb->next_id++;
|
|
}
|
|
|
|
/* Swap-remove an edge from a pointer array by ID. */
|
|
static void remove_edge_from_ptr_array(edge_ptr_array_t *arr, int64_t edge_id) {
|
|
if (!arr) {
|
|
return;
|
|
}
|
|
for (int j = 0; j < arr->count; j++) {
|
|
if (arr->items[j]->id == edge_id) {
|
|
arr->items[j] = arr->items[--arr->count];
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Swap-remove a node from a node_ptr_array by ID. */
|
|
static void remove_node_from_ptr_array(node_ptr_array_t *arr, int64_t node_id) {
|
|
if (!arr) {
|
|
return;
|
|
}
|
|
for (int j = 0; j < arr->count; j++) {
|
|
if (arr->items[j]->id == node_id) {
|
|
arr->items[j] = arr->items[--arr->count];
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Remove an edge from all indexes (dedup + source_type + target_type + type). */
|
|
static void unindex_edge(cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e) {
|
|
char key[EDGE_KEY_BUF];
|
|
|
|
make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type, e->properties_json);
|
|
uint64_t h0;
|
|
uint64_t h1;
|
|
edge_key_hash(key, &h0, &h1);
|
|
edge_key_map_delete(&gb->edge_by_key, h0, h1);
|
|
|
|
make_src_type_key(key, sizeof(key), e->source_id, e->type);
|
|
remove_edge_from_ptr_array(cbm_ht_get(gb->edges_by_source_type, key), e->id);
|
|
|
|
make_src_type_key(key, sizeof(key), e->target_id, e->type);
|
|
remove_edge_from_ptr_array(cbm_ht_get(gb->edges_by_target_type, key), e->id);
|
|
|
|
remove_edge_from_ptr_array(cbm_ht_get(gb->edges_by_type, e->type), e->id);
|
|
}
|
|
|
|
/* Cascade-delete all edges touching nodes in deleted_set. */
|
|
static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) {
|
|
int write_idx = 0;
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
char src_id[CBM_SZ_32];
|
|
char tgt_id[CBM_SZ_32];
|
|
make_id_key(src_id, sizeof(src_id), e->source_id);
|
|
make_id_key(tgt_id, sizeof(tgt_id), e->target_id);
|
|
|
|
if (cbm_ht_get(deleted_set, src_id) || cbm_ht_get(deleted_set, tgt_id)) {
|
|
unindex_edge(gb, e);
|
|
free_edge_strings(e);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e);
|
|
} else {
|
|
gb->edges.items[write_idx++] = gb->edges.items[i];
|
|
}
|
|
}
|
|
gb->edges.count = write_idx;
|
|
}
|
|
|
|
/* Register a node in primary (QN, ID) and secondary (label, name) indexes. */
|
|
static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) {
|
|
cbm_ht_set(gb->node_by_qn, node->qualified_name, node);
|
|
|
|
if (!gb->by_id_off) {
|
|
if (node->id >= gb->by_id_cap) {
|
|
int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;
|
|
while (nc <= node->id) {
|
|
nc *= 2;
|
|
}
|
|
cbm_gbuf_node_t **grown =
|
|
cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown));
|
|
if (grown) {
|
|
memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));
|
|
gb->by_id = grown;
|
|
gb->by_id_cap = nc;
|
|
}
|
|
}
|
|
if (node->id >= 0 && node->id < gb->by_id_cap) {
|
|
gb->by_id[node->id] = node;
|
|
}
|
|
}
|
|
|
|
node_ptr_array_t *by_label =
|
|
get_or_create_node_array(gb->nodes_by_label, node->label ? node->label : "");
|
|
node_array_push(by_label, node);
|
|
|
|
node_ptr_array_t *by_name =
|
|
get_or_create_node_array(gb->nodes_by_name, node->name ? node->name : "");
|
|
node_array_push(by_name, node);
|
|
}
|
|
|
|
/* Per-key index arrays start at two slots: on the kernel the secondary
|
|
* indexes hold ~18M of these arrays and most keys have one or two entries,
|
|
* so the default 8-slot start was 1 GB of empty headroom. NULL = the buffer
|
|
* has no secondary indexes (worker buffers). */
|
|
enum { GB_INDEX_ARRAY_FIRST_CAP = 2 };
|
|
static void edge_array_push(edge_ptr_array_t *arr, const cbm_gbuf_edge_t *edge) {
|
|
if (!arr) {
|
|
return;
|
|
}
|
|
cbm_da_push_min(arr, edge, GB_INDEX_ARRAY_FIRST_CAP);
|
|
}
|
|
static void node_array_push(node_ptr_array_t *arr, const cbm_gbuf_node_t *node) {
|
|
if (!arr) {
|
|
return;
|
|
}
|
|
cbm_da_push_min(arr, node, GB_INDEX_ARRAY_FIRST_CAP);
|
|
}
|
|
|
|
/* Index an edge by one key into a hash table bucket. */
|
|
static void index_edge_by_key(CBMHashTable *ht, const char *key, cbm_gbuf_edge_t *edge) {
|
|
edge_ptr_array_t *arr = get_or_create_edge_array(ht, key);
|
|
edge_array_push(arr, (const cbm_gbuf_edge_t *)edge);
|
|
}
|
|
|
|
/* Register an edge in secondary indexes (source_type, target_type, type). */
|
|
static void register_edge_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_edge_t *edge) {
|
|
char key[EDGE_KEY_BUF];
|
|
|
|
make_src_type_key(key, sizeof(key), edge->source_id, edge->type);
|
|
index_edge_by_key(gb->edges_by_source_type, key, edge);
|
|
|
|
make_src_type_key(key, sizeof(key), edge->target_id, edge->type);
|
|
index_edge_by_key(gb->edges_by_target_type, key, edge);
|
|
|
|
index_edge_by_key(gb->edges_by_type, edge->type, edge);
|
|
}
|
|
|
|
/* Rebuild edge secondary indexes from scratch (after bulk deletion). */
|
|
static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) {
|
|
if (!gb->edges_by_type) {
|
|
return; /* worker buffer: nothing to rebuild */
|
|
}
|
|
cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_source_type);
|
|
cbm_ht_foreach(gb->edges_by_target_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_target_type);
|
|
cbm_ht_foreach(gb->edges_by_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_type);
|
|
|
|
gb->edges_by_source_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
|
|
gb->edges_by_target_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
|
|
gb->edges_by_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32);
|
|
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
register_edge_in_indexes(gb, gb->edges.items[i]);
|
|
}
|
|
}
|
|
|
|
/* Release all lookup hash tables (used by dump after building arrays). */
|
|
static void release_gbuf_indexes(cbm_gbuf_t *gb) {
|
|
cbm_ht_free(gb->node_by_qn);
|
|
gb->node_by_qn = NULL;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
|
|
gb->by_id = NULL;
|
|
gb->by_id_cap = 0;
|
|
cbm_ht_foreach(gb->nodes_by_label, free_node_array, NULL);
|
|
cbm_ht_free(gb->nodes_by_label);
|
|
gb->nodes_by_label = NULL;
|
|
cbm_ht_foreach(gb->nodes_by_name, free_node_array, NULL);
|
|
cbm_ht_free(gb->nodes_by_name);
|
|
gb->nodes_by_name = NULL;
|
|
edge_key_map_free(&gb->edge_by_key);
|
|
cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_source_type);
|
|
gb->edges_by_source_type = NULL;
|
|
cbm_ht_foreach(gb->edges_by_target_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_target_type);
|
|
gb->edges_by_target_type = NULL;
|
|
cbm_ht_foreach(gb->edges_by_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_type);
|
|
gb->edges_by_type = NULL;
|
|
}
|
|
|
|
/* ── Lifecycle ──────────────────────────────────────────────────── */
|
|
|
|
cbm_gbuf_t *cbm_gbuf_new(const char *project, const char *root_path) {
|
|
cbm_gbuf_t *gb = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(cbm_gbuf_t));
|
|
if (!gb) {
|
|
return NULL;
|
|
}
|
|
|
|
gb->project = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, project ? project : "");
|
|
gb->root_path = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, root_path ? root_path : "");
|
|
gb->next_id = SKIP_ONE;
|
|
gb->shared_ids = NULL;
|
|
cbm_arena_init(&gb->proof_arena);
|
|
cbm_resolve_proof_pool_init_project(&gb->proof_pool, &gb->proof_arena);
|
|
|
|
gb->node_by_qn = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
|
|
gb->by_id = NULL;
|
|
gb->by_id_cap = 0;
|
|
gb->nodes_by_label = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32);
|
|
gb->nodes_by_name = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
|
|
|
|
gb->edges_by_source_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
|
|
gb->edges_by_target_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
|
|
gb->edges_by_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32);
|
|
|
|
gb->intern_pool = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_STRING, CBM_SZ_1K);
|
|
|
|
return gb;
|
|
}
|
|
|
|
cbm_gbuf_t *cbm_gbuf_new_shared_ids(const char *project, const char *root_path,
|
|
_Atomic int64_t *id_source) {
|
|
cbm_gbuf_t *gb = cbm_gbuf_new(project, root_path);
|
|
if (gb && id_source) {
|
|
gb->shared_ids = id_source;
|
|
}
|
|
return gb;
|
|
}
|
|
|
|
cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path,
|
|
_Atomic int64_t *id_source) {
|
|
cbm_gbuf_t *gb = cbm_gbuf_new_shared_ids(project, root_path, id_source);
|
|
if (!gb) {
|
|
return NULL;
|
|
}
|
|
/* A worker buffer is appended to and merged; it is never asked by label,
|
|
* name or edge type. Its secondary indexes were pure cost: on the kernel
|
|
* the index class peaked at 7.1 GB during resolve against 2.5 GB live in
|
|
* the main buffer, the difference being 18 workers' throwaway indexes.
|
|
* node_by_qn and edge_by_key stay: upsert and dedup need them. */
|
|
cbm_ht_free(gb->nodes_by_label);
|
|
cbm_ht_free(gb->nodes_by_name);
|
|
cbm_ht_free(gb->edges_by_source_type);
|
|
cbm_ht_free(gb->edges_by_target_type);
|
|
cbm_ht_free(gb->edges_by_type);
|
|
gb->nodes_by_label = NULL;
|
|
gb->nodes_by_name = NULL;
|
|
gb->edges_by_source_type = NULL;
|
|
gb->edges_by_target_type = NULL;
|
|
gb->edges_by_type = NULL;
|
|
/* Nor by id: with ids from the shared counter, a dense id -> node array
|
|
* in every worker spans the whole global id space (8.5M ids x 8 B x 18
|
|
* workers on the kernel) and all of them double in the same instant --
|
|
* a 1 GB step between two reads of the memory gate, which is where the
|
|
* 15 GB budget was missed (2026-09-14). Nothing asks a worker buffer by
|
|
* id before the merge; the main buffer answers after it. */
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
|
|
gb->by_id = NULL;
|
|
gb->by_id_cap = 0;
|
|
gb->by_id_off = true;
|
|
return gb;
|
|
}
|
|
|
|
static void gb_free_span_vec(const char *qn, void *value, void *ud);
|
|
|
|
void cbm_gbuf_free(cbm_gbuf_t *gb) {
|
|
if (!gb) {
|
|
return;
|
|
}
|
|
cbm_gbuf_resolution_end(gb);
|
|
cbm_arena_destroy(&gb->proof_arena);
|
|
cbm_da_free(&gb->import_snapshots);
|
|
cbm_ht_free(gb->imports_by_file);
|
|
cbm_da_free(&gb->resolver_files);
|
|
cbm_da_free(&gb->resolver_summaries);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->resolutions);
|
|
cbm_gbuf_free_captured_inputs(gb->captured_inputs, gb->captured_input_count);
|
|
cbm_ht_free(gb->captured_input_paths);
|
|
if (gb->variant_spans) {
|
|
cbm_ht_foreach(gb->variant_spans, gb_free_span_vec, NULL);
|
|
cbm_ht_free(gb->variant_spans);
|
|
}
|
|
|
|
/* Free each individually-allocated node */
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
free_node_strings(n);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_NODE, n);
|
|
}
|
|
cbm_da_free(&gb->nodes);
|
|
|
|
/* Free each individually-allocated edge */
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
free_edge_strings(e);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e);
|
|
}
|
|
cbm_da_free(&gb->edges);
|
|
|
|
/* Free hash tables — may be NULL if already released by dump_to_sqlite */
|
|
if (gb->node_by_qn) {
|
|
cbm_ht_free(gb->node_by_qn);
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
|
|
if (gb->nodes_by_label) {
|
|
cbm_ht_foreach(gb->nodes_by_label, free_node_array, NULL);
|
|
cbm_ht_free(gb->nodes_by_label);
|
|
}
|
|
if (gb->nodes_by_name) {
|
|
cbm_ht_foreach(gb->nodes_by_name, free_node_array, NULL);
|
|
cbm_ht_free(gb->nodes_by_name);
|
|
}
|
|
edge_key_map_free(&gb->edge_by_key);
|
|
if (gb->edges_by_source_type) {
|
|
cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_source_type);
|
|
}
|
|
if (gb->edges_by_target_type) {
|
|
cbm_ht_foreach(gb->edges_by_target_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_target_type);
|
|
}
|
|
if (gb->edges_by_type) {
|
|
cbm_ht_foreach(gb->edges_by_type, free_edge_array, NULL);
|
|
cbm_ht_free(gb->edges_by_type);
|
|
}
|
|
|
|
/* Free vector storage */
|
|
for (int i = 0; i < gb->dump_vector_count; i++) {
|
|
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_vectors[i].vector);
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_SEMANTIC, gb->dump_vectors);
|
|
|
|
/* Free token vector storage */
|
|
for (int i = 0; i < gb->dump_token_vec_count; i++) {
|
|
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_token_vecs[i].token);
|
|
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_token_vecs[i].vector);
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_SEMANTIC, gb->dump_token_vecs);
|
|
|
|
/* Free interned strings (node label/file_path, edge type) — pool owns one
|
|
* copy each (key == value), freed exactly once via free_intern_entry. Done
|
|
* after nodes/edges since they borrowed these pointers. */
|
|
if (gb->intern_pool) {
|
|
cbm_ht_foreach(gb->intern_pool, free_intern_entry, NULL);
|
|
cbm_ht_free(gb->intern_pool);
|
|
}
|
|
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, gb->project);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, gb->root_path);
|
|
cbm_free(CBM_MEM_CLASS_OTHER, gb);
|
|
}
|
|
|
|
/* ── Vector storage ──────────────────────────────────────────────── */
|
|
|
|
int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector, int vector_len) {
|
|
if (!gb || !vector || vector_len <= 0) {
|
|
return GB_ERR;
|
|
}
|
|
enum { VEC_INIT_CAP = 1024, VEC_GROW = 2 };
|
|
if (gb->dump_vector_count >= gb->dump_vector_cap) {
|
|
int new_cap =
|
|
gb->dump_vector_cap < VEC_INIT_CAP ? VEC_INIT_CAP : gb->dump_vector_cap * VEC_GROW;
|
|
CBMDumpVector *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, gb->dump_vectors,
|
|
(size_t)new_cap * sizeof(CBMDumpVector));
|
|
if (!grown) {
|
|
return GB_ERR;
|
|
}
|
|
gb->dump_vectors = grown;
|
|
gb->dump_vector_cap = new_cap;
|
|
}
|
|
/* Copy vector data */
|
|
uint8_t *vec_copy = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)vector_len);
|
|
if (!vec_copy) {
|
|
return GB_ERR;
|
|
}
|
|
memcpy(vec_copy, vector, (size_t)vector_len);
|
|
|
|
gb->dump_vectors[gb->dump_vector_count++] = (CBMDumpVector){
|
|
.node_id = node_id,
|
|
.project = gb->project, /* borrowed — valid until gbuf_free */
|
|
.vector = vec_copy,
|
|
.vector_len = vector_len,
|
|
};
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t *vector,
|
|
int vector_len, float idf) {
|
|
if (!gb || !token || !vector || vector_len <= 0) {
|
|
return GB_ERR;
|
|
}
|
|
enum { TV_INIT_CAP = 256, TV_GROW = 2 };
|
|
if (gb->dump_token_vec_count >= gb->dump_token_vec_cap) {
|
|
int new_cap =
|
|
gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP : gb->dump_token_vec_cap * TV_GROW;
|
|
CBMDumpTokenVec *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, gb->dump_token_vecs,
|
|
(size_t)new_cap * sizeof(CBMDumpTokenVec));
|
|
if (!grown) {
|
|
return GB_ERR;
|
|
}
|
|
gb->dump_token_vecs = grown;
|
|
gb->dump_token_vec_cap = new_cap;
|
|
}
|
|
uint8_t *vec_copy = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)vector_len);
|
|
if (!vec_copy) {
|
|
return GB_ERR;
|
|
}
|
|
memcpy(vec_copy, vector, (size_t)vector_len);
|
|
|
|
int idx = gb->dump_token_vec_count;
|
|
gb->dump_token_vecs[idx] = (CBMDumpTokenVec){
|
|
.id = idx + SKIP_ONE, /* 1-based sequential ID */
|
|
.project = gb->project,
|
|
.token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token),
|
|
.vector = vec_copy,
|
|
.vector_len = vector_len,
|
|
.idf = idf,
|
|
};
|
|
gb->dump_token_vec_count++;
|
|
return 0;
|
|
}
|
|
|
|
/* ── ID accessors ────────────────────────────────────────────────── */
|
|
|
|
int64_t cbm_gbuf_next_id(const cbm_gbuf_t *gb) {
|
|
if (!gb) {
|
|
return SKIP_ONE;
|
|
}
|
|
if (gb->shared_ids) {
|
|
return atomic_load(gb->shared_ids);
|
|
}
|
|
return gb->next_id;
|
|
}
|
|
|
|
void cbm_gbuf_set_next_id(cbm_gbuf_t *gb, int64_t next_id) {
|
|
if (!gb) {
|
|
return;
|
|
}
|
|
gb->next_id = next_id;
|
|
}
|
|
|
|
void cbm_gbuf_raise_next_id(cbm_gbuf_t *gb, int64_t next_id) {
|
|
if (gb && next_id > gb->next_id) {
|
|
gb->next_id = next_id;
|
|
}
|
|
}
|
|
|
|
static int gb_id_cmp(const void *a, const void *b) {
|
|
int64_t x = *(const int64_t *)a;
|
|
int64_t y = *(const int64_t *)b;
|
|
return (x > y) - (x < y);
|
|
}
|
|
|
|
int cbm_gbuf_duplicate_edge_ids(const cbm_gbuf_t *gb) {
|
|
if (!gb || gb->edges.count < 2) {
|
|
return 0;
|
|
}
|
|
int n = gb->edges.count;
|
|
int64_t *ids = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)n * sizeof(*ids));
|
|
if (!ids) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
for (int i = 0; i < n; i++) {
|
|
ids[i] = gb->edges.items[i]->id;
|
|
}
|
|
qsort(ids, (size_t)n, sizeof(*ids), gb_id_cmp);
|
|
int dups = 0;
|
|
for (int i = 1; i < n; i++) {
|
|
dups += ids[i] == ids[i - SKIP_ONE];
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, ids);
|
|
return dups;
|
|
}
|
|
|
|
/* ── Definition variants ─────────────────────────────────────────────
|
|
* One name defined more than once under mutually exclusive conditions —
|
|
* C-family #if/#else, Rust #[cfg], build-tagged files, a definition under an
|
|
* `if` — is ONE identity: one QN, one node, whose edges are the union of
|
|
* every variant's body. The collision rule below keeps one canonical winner
|
|
* for name, file, lines and properties; every variant's span survives in the
|
|
* node's `variants` property ([{"file_path","start_line","end_line"}],
|
|
* sorted, unique), so a change in any variant maps to the node. Recorded only
|
|
* between definitions of the same label with real spans: a different kind
|
|
* sharing the QN (C: struct vs function) is no variant, a span on line 0 is
|
|
* no location (delta-route proxies), and a full tie is the same entity.
|
|
* C-family platform files (foo_win.c / foo_posix.c, src/unix/ vs src/win/)
|
|
* do not collide here: each definition keeps its path QN, and they share a
|
|
* `variant_key` property instead (helpers.h cbm_platform_neutral_qn), which
|
|
* the test-impact walk follows (store_impact.h follow_variant_keys). */
|
|
static void *gb_yy_malloc(void *ctx, size_t size) {
|
|
(void)ctx;
|
|
return cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, size);
|
|
}
|
|
|
|
static void *gb_yy_realloc(void *ctx, void *block, size_t old_size, size_t size) {
|
|
(void)ctx;
|
|
(void)old_size;
|
|
return cbm_realloc(CBM_MEM_CLASS_GBUF_STRING, block, size);
|
|
}
|
|
|
|
static void gb_yy_free(void *ctx, void *block) {
|
|
(void)ctx;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, block);
|
|
}
|
|
|
|
static const yyjson_alc gb_yy_alc = {gb_yy_malloc, gb_yy_realloc, gb_yy_free, NULL};
|
|
|
|
typedef struct {
|
|
const char *file; /* interned in the buffer's pool */
|
|
int start;
|
|
int end;
|
|
} gb_span_t;
|
|
|
|
/* One side of a same-QN collision. */
|
|
typedef struct {
|
|
const char *label;
|
|
const char *props;
|
|
const char *file;
|
|
int start;
|
|
int end;
|
|
} gb_def_t;
|
|
|
|
/* Every definition span that collapsed into one node, gathered while the
|
|
* buffer is built and written into the node's `variants` once, on export
|
|
* (cbm_gbuf_settle_variants). Rewriting a growing JSON list on every
|
|
* collision was quadratic in the collisions of one QN: 7,586 junit
|
|
* `testcase` elements in one file. */
|
|
typedef struct {
|
|
gb_span_t *items;
|
|
size_t count;
|
|
size_t cap;
|
|
} gb_span_vec_t;
|
|
|
|
static bool gb_variant_label(const char *label) {
|
|
static const char *const labels[] = {"Function", "Method", "Class", "Interface", "Enum",
|
|
"Struct", "Trait", "Type", "Macro"};
|
|
for (size_t i = 0; label && i < sizeof(labels) / sizeof(labels[0]); i++) {
|
|
if (strcmp(label, labels[i]) == 0) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static int gb_span_cmp(const void *a, const void *b) {
|
|
const gb_span_t *x = a;
|
|
const gb_span_t *y = b;
|
|
int c = strcmp(x->file, y->file);
|
|
if (c == 0) {
|
|
c = (x->start > y->start) - (x->start < y->start);
|
|
}
|
|
if (c == 0) {
|
|
c = (x->end > y->end) - (x->end < y->end);
|
|
}
|
|
return c;
|
|
}
|
|
|
|
/* The non-empty `variants` list of a properties document, or NULL. */
|
|
static yyjson_val *gb_variants_list(yyjson_doc *doc) {
|
|
yyjson_val *list = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "variants") : NULL;
|
|
return yyjson_is_arr(list) && yyjson_arr_size(list) > 0 ? list : NULL;
|
|
}
|
|
|
|
static bool gb_props_have_variants(const char *props) {
|
|
return props && strstr(props, "\"variants\"") != NULL;
|
|
}
|
|
|
|
/* Does this side carry a span: a variants list, or a definition's own span? */
|
|
static bool gb_def_has_spans(const gb_def_t *d) {
|
|
return gb_props_have_variants(d->props) || (gb_variant_label(d->label) && d->start > 0);
|
|
}
|
|
|
|
static void gb_span_push(gb_span_vec_t *vec, gb_span_t span) {
|
|
if (vec->count == vec->cap) {
|
|
enum { GB_SPANS_MIN = 4 };
|
|
size_t cap = vec->cap ? vec->cap * PAIR_LEN : GB_SPANS_MIN;
|
|
gb_span_t *grown = cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, vec->items, cap * sizeof(*grown));
|
|
if (!grown) {
|
|
return; /* a lost span costs precision, never data */
|
|
}
|
|
vec->items = grown;
|
|
vec->cap = cap;
|
|
}
|
|
vec->items[vec->count++] = span;
|
|
}
|
|
|
|
/* Append a side's spans: its `variants` list, else its own span when it is a
|
|
* definition. File paths are interned, so the spans outlive the props. */
|
|
static void gb_collect_spans(cbm_gbuf_t *gb, gb_span_vec_t *vec, const gb_def_t *d) {
|
|
if (gb_props_have_variants(d->props)) {
|
|
yyjson_doc *doc = yyjson_read_opts((char *)d->props, strlen(d->props), 0, &gb_yy_alc, NULL);
|
|
yyjson_val *list = gb_variants_list(doc);
|
|
if (list) {
|
|
size_t i;
|
|
size_t max;
|
|
yyjson_val *v;
|
|
yyjson_arr_foreach(list, i, max, v) {
|
|
const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path"));
|
|
int st = (int)yyjson_get_int(yyjson_obj_get(v, "start_line"));
|
|
int en = (int)yyjson_get_int(yyjson_obj_get(v, "end_line"));
|
|
if (f && st > 0) {
|
|
gb_span_push(vec, (gb_span_t){gb_intern(gb, f), st, en});
|
|
}
|
|
}
|
|
yyjson_doc_free(doc);
|
|
return;
|
|
}
|
|
yyjson_doc_free(doc);
|
|
}
|
|
if (gb_variant_label(d->label) && d->start > 0) {
|
|
gb_span_push(vec, (gb_span_t){gb_intern(gb, d->file ? d->file : ""), d->start, d->end});
|
|
}
|
|
}
|
|
|
|
/* Append spans gathered in another buffer (a worker's), re-interned here. */
|
|
static void gb_append_spans(cbm_gbuf_t *gb, gb_span_vec_t *vec, const gb_span_vec_t *from) {
|
|
for (size_t i = 0; i < from->count; i++) {
|
|
gb_span_t span = from->items[i];
|
|
span.file = gb_intern(gb, span.file);
|
|
gb_span_push(vec, span);
|
|
}
|
|
}
|
|
|
|
/* A same-QN collision: gather both sides' spans for the surviving node,
|
|
* whichever label wins and however many there are. Call it before the
|
|
* winner is decided, while `existing` still holds its own fields.
|
|
* `arrival_spans` are the arrival's spans gathered in a worker buffer (they
|
|
* already include its own); NULL when it brings none. */
|
|
static void gb_note_collision(cbm_gbuf_t *gb, const cbm_gbuf_node_t *existing,
|
|
const gb_def_t *arrival, const gb_span_vec_t *arrival_spans) {
|
|
gb_def_t prior = {existing->label, existing->properties_json, existing->file_path,
|
|
existing->start_line, existing->end_line};
|
|
bool arrival_has = arrival_spans ? arrival_spans->count > 0 : gb_def_has_spans(arrival);
|
|
if (!existing->qualified_name || (!gb_def_has_spans(&prior) && !arrival_has)) {
|
|
return;
|
|
}
|
|
if (!gb->variant_spans) {
|
|
gb->variant_spans = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 0);
|
|
if (!gb->variant_spans) {
|
|
return;
|
|
}
|
|
}
|
|
gb_span_vec_t *vec = cbm_ht_get(gb->variant_spans, existing->qualified_name);
|
|
if (!vec) {
|
|
vec = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(*vec));
|
|
if (!vec) {
|
|
return;
|
|
}
|
|
gb_collect_spans(gb, vec, &prior);
|
|
cbm_ht_set(gb->variant_spans, existing->qualified_name, vec);
|
|
}
|
|
if (arrival_spans) {
|
|
gb_append_spans(gb, vec, arrival_spans);
|
|
} else {
|
|
gb_collect_spans(gb, vec, arrival);
|
|
}
|
|
}
|
|
|
|
/* A worker node copied into this buffer brings the spans it gathered there. */
|
|
static void gb_adopt_spans(cbm_gbuf_t *gb, const cbm_gbuf_node_t *node, const gb_span_vec_t *from) {
|
|
if (!node || !node->qualified_name || !from || from->count == 0) {
|
|
return;
|
|
}
|
|
if (!gb->variant_spans) {
|
|
gb->variant_spans = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 0);
|
|
if (!gb->variant_spans) {
|
|
return;
|
|
}
|
|
}
|
|
gb_span_vec_t *vec = cbm_ht_get(gb->variant_spans, node->qualified_name);
|
|
if (!vec) {
|
|
vec = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(*vec));
|
|
if (!vec) {
|
|
return;
|
|
}
|
|
cbm_ht_set(gb->variant_spans, node->qualified_name, vec);
|
|
}
|
|
gb_append_spans(gb, vec, from);
|
|
}
|
|
|
|
static void gb_span_vec_free(gb_span_vec_t *vec) {
|
|
if (vec) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, vec->items);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, vec);
|
|
}
|
|
}
|
|
|
|
/* A node leaves the QN index: its gathered spans go with it, so a later node
|
|
* of the same QN starts from its own. */
|
|
static void gb_drop_variant_spans(cbm_gbuf_t *gb, const char *qn) {
|
|
if (gb->variant_spans && qn) {
|
|
gb_span_vec_free(cbm_ht_delete(gb->variant_spans, qn));
|
|
}
|
|
}
|
|
|
|
/* Write a node's gathered spans into its properties: sorted, distinct, the
|
|
* list left out only when it would say no more than the node's own span. */
|
|
static void gb_write_variants(cbm_gbuf_node_t *node, gb_span_vec_t *vec) {
|
|
qsort(vec->items, vec->count, sizeof(vec->items[0]), gb_span_cmp);
|
|
size_t unique = 0;
|
|
for (size_t i = 0; i < vec->count; i++) {
|
|
if (unique == 0 || gb_span_cmp(&vec->items[unique - SKIP_ONE], &vec->items[i]) != 0) {
|
|
vec->items[unique++] = vec->items[i];
|
|
}
|
|
}
|
|
gb_span_t own = {node->file_path ? node->file_path : "", node->start_line, node->end_line};
|
|
bool own_is_span = gb_variant_label(node->label) && node->start_line > 0;
|
|
bool only_own =
|
|
own_is_span ? unique == SKIP_ONE && gb_span_cmp(&vec->items[0], &own) == 0 : unique == 0;
|
|
if (only_own && !gb_props_have_variants(node->properties_json)) {
|
|
return;
|
|
}
|
|
/* Numbers stay as written: a pass printed them (importance 0.000000). */
|
|
const char *props = node->properties_json;
|
|
yyjson_doc *doc = props ? yyjson_read_opts((char *)props, strlen(props),
|
|
YYJSON_READ_NUMBER_AS_RAW, &gb_yy_alc, NULL)
|
|
: NULL;
|
|
yyjson_mut_doc *mut = yyjson_mut_doc_new(&gb_yy_alc);
|
|
yyjson_mut_val *root = NULL;
|
|
if (mut) {
|
|
root = doc && yyjson_is_obj(yyjson_doc_get_root(doc))
|
|
? yyjson_val_mut_copy(mut, yyjson_doc_get_root(doc))
|
|
: yyjson_mut_obj(mut);
|
|
}
|
|
yyjson_mut_val *list = root && !only_own ? yyjson_mut_arr(mut) : NULL;
|
|
for (size_t i = 0; list && i < unique; i++) {
|
|
yyjson_mut_val *v = yyjson_mut_arr_add_obj(mut, list);
|
|
yyjson_mut_obj_add_strcpy(mut, v, "file_path", vec->items[i].file);
|
|
yyjson_mut_obj_add_int(mut, v, "start_line", vec->items[i].start);
|
|
yyjson_mut_obj_add_int(mut, v, "end_line", vec->items[i].end);
|
|
}
|
|
if (root && (only_own || list)) {
|
|
yyjson_mut_obj_remove_key(root, "variants");
|
|
if (list) {
|
|
yyjson_mut_obj_add_val(mut, root, "variants", list);
|
|
}
|
|
yyjson_mut_doc_set_root(mut, root);
|
|
char *out = yyjson_mut_write_opts(mut, 0, &gb_yy_alc, NULL, NULL);
|
|
if (out) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->properties_json);
|
|
node->properties_json = out;
|
|
}
|
|
}
|
|
yyjson_mut_doc_free(mut);
|
|
yyjson_doc_free(doc);
|
|
}
|
|
|
|
static void gb_settle_one(const char *qn, void *value, void *ud) {
|
|
cbm_gbuf_t *gb = ud;
|
|
gb_span_vec_t *vec = value;
|
|
cbm_gbuf_node_t *node = gb->node_by_qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL;
|
|
if (node && vec) {
|
|
gb_write_variants(node, vec);
|
|
}
|
|
gb_span_vec_free(vec);
|
|
}
|
|
|
|
static void gb_free_span_vec(const char *qn, void *value, void *ud) {
|
|
(void)qn;
|
|
(void)ud;
|
|
gb_span_vec_free(value);
|
|
}
|
|
|
|
void cbm_gbuf_foreach_span(const cbm_gbuf_t *gb, const cbm_gbuf_node_t *n, cbm_gbuf_span_fn fn,
|
|
void *ud) {
|
|
if (!gb || !n || !fn) {
|
|
return;
|
|
}
|
|
const gb_span_vec_t *vec = gb->variant_spans && n->qualified_name
|
|
? cbm_ht_get(gb->variant_spans, n->qualified_name)
|
|
: NULL;
|
|
if (vec && vec->count > 0) {
|
|
for (size_t i = 0; i < vec->count; i++) {
|
|
fn(vec->items[i].file, vec->items[i].start, vec->items[i].end, ud);
|
|
}
|
|
return;
|
|
}
|
|
if (gb_props_have_variants(n->properties_json)) {
|
|
yyjson_doc *doc = yyjson_read_opts((char *)n->properties_json, strlen(n->properties_json),
|
|
0, &gb_yy_alc, NULL);
|
|
yyjson_val *list = gb_variants_list(doc);
|
|
size_t i;
|
|
size_t max;
|
|
yyjson_val *v;
|
|
bool any = false;
|
|
yyjson_arr_foreach(list, i, max, v) {
|
|
const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path"));
|
|
int s = (int)yyjson_get_int(yyjson_obj_get(v, "start_line"));
|
|
int e = (int)yyjson_get_int(yyjson_obj_get(v, "end_line"));
|
|
if (f && s > 0) {
|
|
fn(f, s, e, ud);
|
|
any = true;
|
|
}
|
|
}
|
|
yyjson_doc_free(doc);
|
|
if (any) {
|
|
return;
|
|
}
|
|
}
|
|
if (n->file_path && n->start_line > 0) {
|
|
fn(n->file_path, n->start_line, n->end_line, ud);
|
|
}
|
|
}
|
|
|
|
void cbm_gbuf_settle_variants(cbm_gbuf_t *gb) {
|
|
if (!gb || !gb->variant_spans) {
|
|
return;
|
|
}
|
|
cbm_ht_foreach(gb->variant_spans, gb_settle_one, gb);
|
|
cbm_ht_free(gb->variant_spans);
|
|
gb->variant_spans = NULL;
|
|
}
|
|
|
|
/* ── Node operations ─────────────────────────────────────────────── */
|
|
|
|
/* First key of the canonical same-QN collision order: how strong a claim the
|
|
* entity has on a qualified name. A macro is the weakest: when a "Macro" and
|
|
* a definition share a QN, the definition, which carries the body, the
|
|
* signature and every edge, owns it whichever comes first.
|
|
*
|
|
* C-preprocessor macros no longer get here: their QN carries its own
|
|
* namespace (`...#macro`, CBM_MACRO_QN_SUFFIX) and cannot equal a definition's.
|
|
* The case left is Chialisp, whose `defmacro` / `defmac` defs are labelled
|
|
* "Macro" under a plain QN: a macro and a `defun` of one name in one file, or
|
|
* in a same-stem `.clsp` / `.clib` pair (one module QN). */
|
|
static int gb_qn_claim_rank(const char *label) {
|
|
return (label && strcmp(label, "Macro") == 0) ? 0 : 1;
|
|
}
|
|
|
|
int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name,
|
|
const char *qualified_name, const char *file_path, int start_line,
|
|
int end_line, const char *properties_json) {
|
|
if (!gb || !qualified_name) {
|
|
return 0;
|
|
}
|
|
|
|
/* Check if node already exists */
|
|
cbm_gbuf_node_t *existing = cbm_ht_get(gb->node_by_qn, qualified_name);
|
|
if (existing) {
|
|
/* Don't let a per-file "Module" def touch a structural directory node
|
|
* ("Project" root or "Folder"). In a directory-based-module language
|
|
* (Go/Java) a file's module_qn equals its directory QN: a root file →
|
|
* the project name (== the "Project" node's QN); a file in pkg/ →
|
|
* proj.pkg (== the "pkg/" Folder node's QN). Its always-emitted Module
|
|
* def collides here; the directory node is the package/module container
|
|
* and must keep its structural label AND its own name/file_path/range.
|
|
* Updating those in place set the shared node's file_path to whichever
|
|
* same-package file happened to be processed LAST (worker-order
|
|
* dependent) — the nondeterministic file attribution behind #787 — and
|
|
* left the Folder node exposed to delete-nodes-by-file on incremental
|
|
* reindex of that file. Skip the update entirely. (Both the sequential
|
|
* upsert and the parallel local-gbuf merge route through this function.) */
|
|
if (existing->label && label && strcmp(label, "Module") == 0 &&
|
|
(strcmp(existing->label, "Project") == 0 || strcmp(existing->label, "Folder") == 0)) {
|
|
return existing->id;
|
|
}
|
|
/* Same-QN arrival: distinct source entities can share a QN (C: a
|
|
* struct, a function and a macro with one name), and the same entity
|
|
* can be re-upserted with fresh content. The old code let the LAST
|
|
* arrival overwrite — under parallel extraction the merge order
|
|
* varies run to run, so WHICH entity survived flickered (xfs:
|
|
* Function-node count 4998 vs 5015 across two runs) and every
|
|
* order-sensitive consumer downstream inherited it. Pick the
|
|
* survivor by a canonical CONTENT rule instead, a pure function of
|
|
* the two candidates: smallest file_path, then LARGEST start_line,
|
|
* then largest name/label (a mixed-direction composite is still a
|
|
* total order, so the pick is commutative and scheduling-free).
|
|
* Line-descending within a file keeps the classic upsert contract —
|
|
* a later definition in the same file (macro redefinition, refresh
|
|
* of the same entity with new content) replaces the earlier one,
|
|
* deterministically, because intra-file arrival order is fixed. A
|
|
* full tie is the same entity re-upserted → refresh in place.
|
|
* Ahead of all of these, a definition beats a macro
|
|
* (gb_qn_claim_rank). Kind-disambiguated QNs (the real cure) remain
|
|
* a follow-up. */
|
|
int c = gb_qn_claim_rank(existing->label) - gb_qn_claim_rank(label);
|
|
if (c == 0) {
|
|
c = strcmp(file_path ? file_path : "", existing->file_path ? existing->file_path : "");
|
|
}
|
|
if (c == 0) {
|
|
c = existing->start_line - start_line;
|
|
}
|
|
if (c == 0) {
|
|
c = strcmp(existing->name ? existing->name : "", name ? name : "");
|
|
}
|
|
if (c == 0) {
|
|
c = strcmp(existing->label ? existing->label : "", label ? label : "");
|
|
}
|
|
gb_def_t arrival = {label, properties_json, file_path, start_line, end_line};
|
|
gb_note_collision(gb, existing, &arrival, NULL);
|
|
if (c > 0) {
|
|
return existing->id; /* existing entity is the canonical winner */
|
|
}
|
|
/* The arrival wins: its properties; the variant spans both carry are
|
|
* written on export (cbm_gbuf_settle_variants). */
|
|
/* Update in-place. name/properties are strdup'd BEFORE freeing old ones
|
|
* (callers may pass existing->name as an argument). label/file_path are
|
|
* interned: gb_intern returns a stable pool pointer (idempotent even when
|
|
* label == existing->label), so the old value is replaced, never freed.
|
|
* When the surviving label/name changes, keep the secondary indexes
|
|
* consistent (the old code left the node listed under its ORIGINAL
|
|
* label/name — cbm_gbuf_find_by_label/name then missed or mis-listed it,
|
|
* which is how the flickering Function set reached the semantic pass). */
|
|
char *new_name = heap_strdup(name);
|
|
char *new_props = properties_json ? heap_strdup(properties_json) : NULL;
|
|
const char *new_label_interned = gb_intern(gb, label);
|
|
bool label_changed = !existing->label || !new_label_interned ||
|
|
strcmp(existing->label, new_label_interned) != 0;
|
|
bool name_changed = !existing->name || !new_name || strcmp(existing->name, new_name) != 0;
|
|
if (label_changed) {
|
|
remove_node_from_ptr_array(
|
|
cbm_ht_get(gb->nodes_by_label, existing->label ? existing->label : ""),
|
|
existing->id);
|
|
}
|
|
if (name_changed) {
|
|
remove_node_from_ptr_array(
|
|
cbm_ht_get(gb->nodes_by_name, existing->name ? existing->name : ""), existing->id);
|
|
}
|
|
existing->label = (char *)new_label_interned;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->name);
|
|
existing->name = new_name;
|
|
existing->file_path = (char *)gb_intern(gb, file_path);
|
|
existing->start_line = start_line;
|
|
existing->end_line = end_line;
|
|
if (new_props) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json);
|
|
existing->properties_json = new_props;
|
|
}
|
|
if (label_changed) {
|
|
node_ptr_array_t *by_label = get_or_create_node_array(
|
|
gb->nodes_by_label, existing->label ? existing->label : "");
|
|
node_array_push(by_label, existing);
|
|
}
|
|
if (name_changed) {
|
|
node_ptr_array_t *by_name =
|
|
get_or_create_node_array(gb->nodes_by_name, existing->name ? existing->name : "");
|
|
node_array_push(by_name, existing);
|
|
}
|
|
return existing->id;
|
|
}
|
|
|
|
/* Heap-allocate a new node (pointer stays stable across array growth) */
|
|
cbm_gbuf_node_t *node = cbm_calloc(CBM_MEM_CLASS_GBUF_NODE, sizeof(cbm_gbuf_node_t));
|
|
if (!node) {
|
|
return 0;
|
|
}
|
|
|
|
int64_t id = alloc_next_id(gb);
|
|
node->id = id;
|
|
node->label = (char *)gb_intern(gb, label);
|
|
node->name = heap_strdup(name);
|
|
node->qualified_name = heap_strdup(qualified_name);
|
|
node->file_path = (char *)gb_intern(gb, file_path);
|
|
node->start_line = start_line;
|
|
node->end_line = end_line;
|
|
node->properties_json = heap_strdup(properties_json);
|
|
|
|
/* Store pointer in array and register in all indexes */
|
|
cbm_da_push(&gb->nodes, node);
|
|
register_node_in_indexes(gb, node);
|
|
|
|
return id;
|
|
}
|
|
|
|
int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json) {
|
|
if (!node) {
|
|
return GB_ERR;
|
|
}
|
|
char *copy = heap_strdup(json);
|
|
if (!copy) {
|
|
return GB_ERR;
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->properties_json);
|
|
node->properties_json = copy;
|
|
return 0;
|
|
}
|
|
|
|
const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn) {
|
|
if (!gb || !qn) {
|
|
return NULL;
|
|
}
|
|
return cbm_ht_get(gb->node_by_qn, qn);
|
|
}
|
|
|
|
const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) {
|
|
if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) {
|
|
return NULL;
|
|
}
|
|
return gb->by_id[id];
|
|
}
|
|
|
|
int cbm_gbuf_find_by_label(const cbm_gbuf_t *gb, const char *label, const cbm_gbuf_node_t ***out,
|
|
int *count) {
|
|
if (!gb || !out || !count) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label ? label : "");
|
|
if (arr && arr->count > 0) {
|
|
*out = arr->items;
|
|
*count = arr->count;
|
|
} else {
|
|
*out = NULL;
|
|
*count = 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_find_by_name(const cbm_gbuf_t *gb, const char *name, const cbm_gbuf_node_t ***out,
|
|
int *count) {
|
|
if (!gb || !out || !count) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_name, name ? name : "");
|
|
if (arr && arr->count > 0) {
|
|
*out = arr->items;
|
|
*count = arr->count;
|
|
} else {
|
|
*out = NULL;
|
|
*count = 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_node_count(const cbm_gbuf_t *gb) {
|
|
/* Use QN hash table count since it's authoritative (handles deletes) */
|
|
return gb ? (int)cbm_ht_count(gb->node_by_qn) : 0;
|
|
}
|
|
|
|
int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) {
|
|
if (!gb || !label) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label);
|
|
if (!arr || arr->count == 0) {
|
|
return 0;
|
|
}
|
|
|
|
/* Build hash set of deleted node IDs for O(1) lookup */
|
|
CBMHashTable *deleted_set = cbm_ht_create(arr->count);
|
|
for (int i = 0; i < arr->count; i++) {
|
|
const cbm_gbuf_node_t *n = arr->items[i];
|
|
|
|
char id_buf[CBM_SZ_32];
|
|
make_id_key(id_buf, sizeof(id_buf), n->id);
|
|
cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf),
|
|
intptr_to_ptr(SKIP_ONE));
|
|
|
|
/* Remove from primary indexes */
|
|
gb_drop_variant_spans(gb, n->qualified_name);
|
|
cbm_ht_delete(gb->node_by_qn, n->qualified_name);
|
|
if (n->id >= 0 && n->id < gb->by_id_cap) {
|
|
gb->by_id[n->id] = NULL;
|
|
}
|
|
}
|
|
|
|
/* Clear the label array */
|
|
cbm_da_clear(arr);
|
|
|
|
/* Cascade-delete edges referencing deleted nodes */
|
|
cascade_delete_edges(gb, deleted_set);
|
|
|
|
cbm_ht_foreach(deleted_set, free_index_key, NULL);
|
|
cbm_ht_free(deleted_set);
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) {
|
|
if (!gb || !file_path) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
/* Collect IDs of nodes in this file */
|
|
CBMHashTable *deleted_set = cbm_ht_create(CBM_SZ_64);
|
|
int deleted_count = 0;
|
|
int scanned = 0;
|
|
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
scanned++;
|
|
if (!n->file_path || strcmp(n->file_path, file_path) != 0) {
|
|
continue;
|
|
}
|
|
if (!n->qualified_name || !cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
|
continue;
|
|
}
|
|
|
|
char id_buf[CBM_SZ_32];
|
|
make_id_key(id_buf, sizeof(id_buf), n->id);
|
|
cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf),
|
|
intptr_to_ptr(SKIP_ONE));
|
|
|
|
/* Remove from secondary indexes */
|
|
remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_label, n->label), n->id);
|
|
remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_name, n->name), n->id);
|
|
|
|
/* Remove from primary indexes */
|
|
gb_drop_variant_spans(gb, n->qualified_name);
|
|
cbm_ht_delete(gb->node_by_qn, n->qualified_name);
|
|
if (n->id >= 0 && n->id < gb->by_id_cap) {
|
|
gb->by_id[n->id] = NULL;
|
|
}
|
|
|
|
/* NULL out QN so dump's liveness check (cbm_ht_get by QN) fails
|
|
* even if a new node with the same QN is inserted later via merge. */
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->qualified_name);
|
|
n->qualified_name = NULL;
|
|
deleted_count++;
|
|
}
|
|
|
|
if (deleted_count == 0) {
|
|
cbm_ht_free(deleted_set);
|
|
return 0;
|
|
}
|
|
|
|
/* Cascade-delete edges referencing deleted nodes */
|
|
cascade_delete_edges(gb, deleted_set);
|
|
|
|
cbm_ht_foreach(deleted_set, free_index_key, NULL);
|
|
cbm_ht_free(deleted_set);
|
|
{
|
|
char s_buf[CBM_SZ_16];
|
|
char d_buf[CBM_SZ_16];
|
|
snprintf(s_buf, sizeof(s_buf), "%d", scanned);
|
|
snprintf(d_buf, sizeof(d_buf), "%d", deleted_count);
|
|
cbm_log_info("gbuf.delete_by_file", "file", file_path, "scanned", s_buf, "deleted", d_buf);
|
|
}
|
|
return deleted_count;
|
|
}
|
|
|
|
/* Whether an edge of `types` (NULL-terminated) touches node `id`. */
|
|
static bool gbuf_node_touched(const cbm_gbuf_t *gb, int64_t id, const char *const *types) {
|
|
for (int t = 0; types[t]; t++) {
|
|
const cbm_gbuf_edge_t **edges = NULL;
|
|
int count = 0;
|
|
if ((cbm_gbuf_find_edges_by_target_type(gb, id, types[t], &edges, &count) == 0 &&
|
|
count > 0) ||
|
|
(cbm_gbuf_find_edges_by_source_type(gb, id, types[t], &edges, &count) == 0 &&
|
|
count > 0)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/* Delete the `label` nodes no edge of `types` (NULL-terminated) touches --
|
|
* with `fileless`, only those carrying no file; returns how many. */
|
|
static int gbuf_delete_untouched(cbm_gbuf_t *gb, const char *label, const char *const *types,
|
|
bool fileless) {
|
|
node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label);
|
|
if (!arr || arr->count == 0) {
|
|
return 0;
|
|
}
|
|
/* The victims first: the per-node unlinking below edits `arr`. */
|
|
cbm_gbuf_node_t **victims =
|
|
cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)arr->count * sizeof(*victims));
|
|
if (!victims) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
int n = 0;
|
|
for (int i = 0; i < arr->count; i++) {
|
|
const char *qn = arr->items[i]->qualified_name;
|
|
cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL;
|
|
if (node && !(fileless && node->file_path && node->file_path[0]) &&
|
|
!gbuf_node_touched(gb, node->id, types)) {
|
|
victims[n++] = node;
|
|
}
|
|
}
|
|
if (n == 0) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, victims);
|
|
return 0;
|
|
}
|
|
CBMHashTable *deleted_set = cbm_ht_create((uint32_t)n);
|
|
for (int i = 0; i < n; i++) {
|
|
cbm_gbuf_node_t *node = victims[i];
|
|
if (!node->qualified_name) {
|
|
continue;
|
|
}
|
|
char id_buf[CBM_SZ_32];
|
|
make_id_key(id_buf, sizeof(id_buf), node->id);
|
|
cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf),
|
|
intptr_to_ptr(SKIP_ONE));
|
|
remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_label, node->label), node->id);
|
|
remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_name, node->name), node->id);
|
|
gb_drop_variant_spans(gb, node->qualified_name);
|
|
cbm_ht_delete(gb->node_by_qn, node->qualified_name);
|
|
if (node->id >= 0 && node->id < gb->by_id_cap) {
|
|
gb->by_id[node->id] = NULL;
|
|
}
|
|
/* As delete_by_file: a NULL QN fails the dump's liveness check. */
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->qualified_name);
|
|
node->qualified_name = NULL;
|
|
}
|
|
cascade_delete_edges(gb, deleted_set);
|
|
cbm_ht_foreach(deleted_set, free_index_key, NULL);
|
|
cbm_ht_free(deleted_set);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, victims);
|
|
return n;
|
|
}
|
|
|
|
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) {
|
|
if (!gb || !label || !edge_types || !edge_types[0]) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
return gbuf_delete_untouched(gb, label, edge_types, false);
|
|
}
|
|
|
|
typedef struct {
|
|
const char **items;
|
|
uint32_t count;
|
|
} gb_type_list_t;
|
|
|
|
static void collect_edge_type(const char *key, void *value, void *ud) {
|
|
(void)value;
|
|
gb_type_list_t *list = ud;
|
|
list->items[list->count++] = key;
|
|
}
|
|
|
|
int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label) {
|
|
if (!gb || !label) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
/* Edges are indexed by (node, type) only: "any edge" is an edge of any
|
|
* type the buffer holds (NULL-terminated by the calloc). */
|
|
gb_type_list_t list = {
|
|
cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX,
|
|
((size_t)cbm_ht_count(gb->edges_by_type) + 1) * sizeof(*list.items)),
|
|
0};
|
|
if (!list.items) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
cbm_ht_foreach(gb->edges_by_type, collect_edge_type, &list);
|
|
int n = gbuf_delete_untouched(gb, label, list.items, true);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, list.items);
|
|
return n;
|
|
}
|
|
|
|
int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project) {
|
|
if (!gb || !db_path || !project) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
cbm_store_t *store = cbm_store_open_path(db_path);
|
|
if (!store) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
sqlite3 *db = cbm_store_get_db(store);
|
|
if (!db) {
|
|
cbm_store_close(store);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
/* First pass: find max node ID for mapping array */
|
|
sqlite3_stmt *stmt = NULL;
|
|
if (sqlite3_prepare_v2(db, "SELECT MAX(id) FROM nodes WHERE project = ?", CBM_NOT_FOUND, &stmt,
|
|
NULL) != SQLITE_OK) {
|
|
cbm_store_close(store);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
sqlite3_bind_text(stmt, SKIP_ONE, project, CBM_NOT_FOUND, SQLITE_STATIC);
|
|
int64_t max_old_id = 0;
|
|
if (sqlite3_step(stmt) == SQLITE_ROW) {
|
|
max_old_id = sqlite3_column_int64(stmt, 0);
|
|
}
|
|
sqlite3_finalize(stmt);
|
|
|
|
int64_t *old_to_new =
|
|
cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)(max_old_id + SKIP_ONE) * sizeof(int64_t));
|
|
if (!old_to_new) {
|
|
cbm_store_close(store);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
/* Load all nodes */
|
|
if (sqlite3_prepare_v2(
|
|
db,
|
|
"SELECT id, label, name, qualified_name, file_path, start_line, end_line, properties "
|
|
"FROM nodes WHERE project = ? ORDER BY id",
|
|
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new);
|
|
cbm_store_close(store);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
sqlite3_bind_text(stmt, SKIP_ONE, project, CBM_NOT_FOUND, SQLITE_STATIC);
|
|
|
|
while (sqlite3_step(stmt) == SQLITE_ROW) {
|
|
int64_t old_id = sqlite3_column_int64(stmt, 0);
|
|
const char *label = (const char *)sqlite3_column_text(stmt, SKIP_ONE);
|
|
const char *name = (const char *)sqlite3_column_text(stmt, GB_COL_2);
|
|
const char *qn = (const char *)sqlite3_column_text(stmt, GB_COL_3);
|
|
const char *fp = (const char *)sqlite3_column_text(stmt, GB_COL_4);
|
|
int sl = sqlite3_column_int(stmt, GB_COL_5);
|
|
int el = sqlite3_column_int(stmt, GB_COL_6);
|
|
const char *props = (const char *)sqlite3_column_text(stmt, GB_COL_7);
|
|
|
|
int64_t new_id = cbm_gbuf_upsert_node(gb, label, name, qn, fp, sl, el, props);
|
|
if (new_id > 0 && old_id <= max_old_id) {
|
|
old_to_new[old_id] = new_id;
|
|
}
|
|
}
|
|
sqlite3_finalize(stmt);
|
|
|
|
/* Load all edges, remap IDs */
|
|
if (sqlite3_prepare_v2(db,
|
|
"SELECT source_id, target_id, type, properties "
|
|
"FROM edges WHERE project = ?",
|
|
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new);
|
|
cbm_store_close(store);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
sqlite3_bind_text(stmt, SKIP_ONE, project, CBM_NOT_FOUND, SQLITE_STATIC);
|
|
|
|
while (sqlite3_step(stmt) == SQLITE_ROW) {
|
|
int64_t old_src = sqlite3_column_int64(stmt, 0);
|
|
int64_t old_tgt = sqlite3_column_int64(stmt, SKIP_ONE);
|
|
const char *type = (const char *)sqlite3_column_text(stmt, GB_COL_2);
|
|
const char *props = (const char *)sqlite3_column_text(stmt, GB_COL_3);
|
|
|
|
int64_t new_src = (old_src <= max_old_id) ? old_to_new[old_src] : 0;
|
|
int64_t new_tgt = (old_tgt <= max_old_id) ? old_to_new[old_tgt] : 0;
|
|
if (new_src > 0 && new_tgt > 0) {
|
|
cbm_gbuf_insert_edge(gb, new_src, new_tgt, type, props);
|
|
}
|
|
}
|
|
sqlite3_finalize(stmt);
|
|
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new);
|
|
cbm_store_close(store);
|
|
return 0;
|
|
}
|
|
|
|
void cbm_gbuf_foreach_node(const cbm_gbuf_t *gb, cbm_gbuf_node_visitor_fn fn, void *userdata) {
|
|
if (!gb || !fn) {
|
|
return;
|
|
}
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
const cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
if (n->qualified_name && cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
|
fn(n, userdata);
|
|
}
|
|
}
|
|
}
|
|
|
|
void cbm_gbuf_foreach_edge(const cbm_gbuf_t *gb, cbm_gbuf_edge_visitor_fn fn, void *userdata) {
|
|
if (!gb || !fn) {
|
|
return;
|
|
}
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
fn(gb->edges.items[i], userdata);
|
|
}
|
|
}
|
|
|
|
/* ── Edge operations ─────────────────────────────────────────────── */
|
|
|
|
/* Read "confidence":<double> out of an edge property blob. Absent/unparseable
|
|
* reads as -1 so any edge that carries a confidence outranks one that does
|
|
* not. */
|
|
static double edge_props_confidence(const char *props_json) {
|
|
if (!props_json) {
|
|
return CBM_EDGE_CONF_ABSENT;
|
|
}
|
|
static const char conf_key[] = "\"confidence\":";
|
|
const char *p = strstr(props_json, conf_key);
|
|
if (!p) {
|
|
return CBM_EDGE_CONF_ABSENT;
|
|
}
|
|
/* strtod answers 0.0 for text it cannot read, and 0.0 is a real
|
|
* confidence that beats CBM_EDGE_CONF_ABSENT. So a blob carrying
|
|
* "confidence":null used to outrank a clean blob that carries no
|
|
* confidence at all, and displace it. Ask strtod where it stopped: an
|
|
* end pointer that never moved means it read nothing. */
|
|
const char *value = p + sizeof(conf_key) - SKIP_ONE;
|
|
char *end = NULL;
|
|
double conf = strtod(value, &end);
|
|
if (end == value) {
|
|
return CBM_EDGE_CONF_ABSENT;
|
|
}
|
|
return conf;
|
|
}
|
|
|
|
/* Decide whether an incoming property blob replaces the stored one on a
|
|
* duplicate edge key.
|
|
*
|
|
* confidence/strategy/via are not part of the edge key, and the same logical
|
|
* edge is legitimately minted by more than one strategy carrying different
|
|
* values — LSP resolution and registry-textual matching both produce CALLS
|
|
* edges. Per-worker edge buffers merge in worker-slot order, so any rule that
|
|
* depends on arrival order makes the surviving attributes a function of thread
|
|
* scheduling.
|
|
*
|
|
* This rule is a total order over the two candidates, hence commutative and
|
|
* associative — the outcome is independent of arrival order:
|
|
* 1. higher "confidence" wins, which also enforces the intended precedence
|
|
* that an LSP-resolved call outranks a textual match;
|
|
* 2. on equal confidence, the lexicographically greater blob wins, giving a
|
|
* deterministic choice between otherwise equal candidates.
|
|
* An empty or absent incoming blob never displaces stored properties. */
|
|
static bool edge_props_should_replace(const char *existing_json, const char *incoming_json) {
|
|
if (!incoming_json || strcmp(incoming_json, "{}") == 0) {
|
|
return false;
|
|
}
|
|
if (!existing_json || strcmp(existing_json, "{}") == 0) {
|
|
return true;
|
|
}
|
|
double inc = edge_props_confidence(incoming_json);
|
|
double cur = edge_props_confidence(existing_json);
|
|
if (inc != cur) {
|
|
return inc > cur;
|
|
}
|
|
return strcmp(incoming_json, existing_json) > 0;
|
|
}
|
|
|
|
int64_t cbm_gbuf_insert_edge(cbm_gbuf_t *gb, int64_t source_id, int64_t target_id, const char *type,
|
|
const char *properties_json) {
|
|
if (!gb || !type) {
|
|
return 0;
|
|
}
|
|
|
|
/* Check for dedup */
|
|
char key[EDGE_KEY_BUF];
|
|
make_edge_key(key, sizeof(key), source_id, target_id, type, properties_json);
|
|
|
|
uint64_t h0;
|
|
uint64_t h1;
|
|
edge_key_hash(key, &h0, &h1);
|
|
edge_key_slot_t *hit = edge_key_map_find(&gb->edge_by_key, h0, h1);
|
|
cbm_gbuf_edge_t *existing = hit ? hit->edge : NULL;
|
|
if (existing) {
|
|
if (edge_props_should_replace(existing->properties_json, properties_json)) {
|
|
existing->properties_json = (char *)gb_intern(gb, properties_json);
|
|
}
|
|
return existing->id;
|
|
}
|
|
|
|
/* Heap-allocate a new edge (pointer stays stable) */
|
|
cbm_gbuf_edge_t *edge = cbm_calloc(CBM_MEM_CLASS_GBUF_EDGE, sizeof(cbm_gbuf_edge_t));
|
|
if (!edge) {
|
|
return 0;
|
|
}
|
|
|
|
int64_t id = alloc_next_id(gb);
|
|
edge->id = id;
|
|
edge->source_id = source_id;
|
|
edge->target_id = target_id;
|
|
edge->type = (char *)gb_intern(gb, type);
|
|
edge->properties_json = (char *)gb_intern(gb, properties_json);
|
|
|
|
/* Publication reads this primary array. Do not register a missing edge
|
|
* in secondary indexes or report success when array growth fails. */
|
|
int before = gb->edges.count;
|
|
cbm_da_push(&gb->edges, edge);
|
|
if (gb->edges.count != before + 1) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, edge);
|
|
return 0;
|
|
}
|
|
|
|
/* Dedup index */
|
|
(void)edge_key_map_set(&gb->edge_by_key, h0, h1, edge);
|
|
|
|
/* Secondary indexes */
|
|
register_edge_in_indexes(gb, edge);
|
|
|
|
return id;
|
|
}
|
|
|
|
int cbm_gbuf_find_edges_by_source_type(const cbm_gbuf_t *gb, int64_t source_id, const char *type,
|
|
const cbm_gbuf_edge_t ***out, int *count) {
|
|
if (!gb || !out || !count) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
char key[EDGE_KEY_BUF];
|
|
make_src_type_key(key, sizeof(key), source_id, type);
|
|
edge_ptr_array_t *arr = cbm_ht_get(gb->edges_by_source_type, key);
|
|
if (arr && arr->count > 0) {
|
|
*out = arr->items;
|
|
*count = arr->count;
|
|
} else {
|
|
*out = NULL;
|
|
*count = 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_find_edges_by_target_type(const cbm_gbuf_t *gb, int64_t target_id, const char *type,
|
|
const cbm_gbuf_edge_t ***out, int *count) {
|
|
if (!gb || !out || !count) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
char key[EDGE_KEY_BUF];
|
|
make_src_type_key(key, sizeof(key), target_id, type);
|
|
edge_ptr_array_t *arr = cbm_ht_get(gb->edges_by_target_type, key);
|
|
if (arr && arr->count > 0) {
|
|
*out = arr->items;
|
|
*count = arr->count;
|
|
} else {
|
|
*out = NULL;
|
|
*count = 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_find_edges_by_type(const cbm_gbuf_t *gb, const char *type,
|
|
const cbm_gbuf_edge_t ***out, int *count) {
|
|
if (!gb || !out || !count) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
edge_ptr_array_t *arr = cbm_ht_get(gb->edges_by_type, type);
|
|
if (arr && arr->count > 0) {
|
|
*out = arr->items;
|
|
*count = arr->count;
|
|
} else {
|
|
*out = NULL;
|
|
*count = 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_edge_count(const cbm_gbuf_t *gb) {
|
|
return gb ? gb->edges.count : 0;
|
|
}
|
|
|
|
int cbm_gbuf_edge_count_by_type(const cbm_gbuf_t *gb, const char *type) {
|
|
if (!gb || !type) {
|
|
return 0;
|
|
}
|
|
edge_ptr_array_t *arr = cbm_ht_get(gb->edges_by_type, type);
|
|
return arr ? arr->count : 0;
|
|
}
|
|
|
|
/* Remove the edges `drop` selects from the array and the dedup index, then
|
|
* rebuild the secondary indexes. */
|
|
typedef bool (*gbuf_edge_pred_t)(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud);
|
|
|
|
static int gbuf_delete_edges_if(cbm_gbuf_t *gb, gbuf_edge_pred_t drop, const void *ud) {
|
|
int write_idx = 0;
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
if (drop(gb, e, ud)) {
|
|
char key[EDGE_KEY_BUF];
|
|
make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type,
|
|
e->properties_json);
|
|
uint64_t h0;
|
|
uint64_t h1;
|
|
edge_key_hash(key, &h0, &h1);
|
|
edge_key_map_delete(&gb->edge_by_key, h0, h1);
|
|
free_edge_strings(e);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e);
|
|
} else {
|
|
gb->edges.items[write_idx++] = gb->edges.items[i];
|
|
}
|
|
}
|
|
gb->edges.count = write_idx;
|
|
|
|
/* Rebuild edge secondary indexes */
|
|
rebuild_edge_secondary_indexes(gb);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static bool gbuf_edge_is_type(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) {
|
|
(void)gb;
|
|
return strcmp(e->type, (const char *)ud) == 0;
|
|
}
|
|
|
|
int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) {
|
|
if (!gb || !type) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
return gbuf_delete_edges_if(gb, gbuf_edge_is_type, type);
|
|
}
|
|
|
|
typedef struct {
|
|
const char *type;
|
|
const char *label;
|
|
} gbuf_type_label_t;
|
|
|
|
static bool gbuf_edge_from_label(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) {
|
|
const gbuf_type_label_t *tl = ud;
|
|
if (strcmp(e->type, tl->type) != 0) {
|
|
return false;
|
|
}
|
|
const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(gb, e->source_id);
|
|
return src && src->label && strcmp(src->label, tl->label) == 0;
|
|
}
|
|
|
|
int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const char *label) {
|
|
if (!gb || !type || !label) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
gbuf_type_label_t tl = {type, label};
|
|
return gbuf_delete_edges_if(gb, gbuf_edge_from_label, &tl);
|
|
}
|
|
|
|
static int gbuf_ptr_cmp(const void *k, const void *v) {
|
|
uintptr_t a = (uintptr_t)*(const cbm_gbuf_edge_t *const *)k;
|
|
uintptr_t b = (uintptr_t)*(const cbm_gbuf_edge_t *const *)v;
|
|
return (a > b) - (a < b);
|
|
}
|
|
|
|
typedef struct {
|
|
const cbm_gbuf_edge_t **edges;
|
|
int n;
|
|
} gbuf_edge_set_t;
|
|
|
|
static bool gbuf_edge_in_set(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) {
|
|
(void)gb;
|
|
const gbuf_edge_set_t *s = ud;
|
|
return bsearch(&e, s->edges, (size_t)s->n, sizeof(*s->edges), gbuf_ptr_cmp) != NULL;
|
|
}
|
|
|
|
/* By the edges themselves, not their ids: after the parallel resolve merge,
|
|
* cbm_gbuf_set_next_id can hand an id out twice. */
|
|
int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n) {
|
|
if (!gb || !edges || n <= 0) {
|
|
return 0;
|
|
}
|
|
qsort((void *)edges, (size_t)n, sizeof(*edges), gbuf_ptr_cmp);
|
|
gbuf_edge_set_t s = {edges, n};
|
|
return gbuf_delete_edges_if(gb, gbuf_edge_in_set, &s);
|
|
}
|
|
|
|
/* ── Merge ───────────────────────────────────────────────────────── */
|
|
|
|
/* Free remap hash table entries (key = heap string, value = heap int64_t*) */
|
|
static void free_remap_entry(const char *key, void *val, void *ud) {
|
|
(void)ud;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
|
|
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, val);
|
|
}
|
|
|
|
/* Handle QN collision: update dst node fields (src wins), record remap if IDs differ.
|
|
* label/file_path are re-interned into dst's pool (sn's pointers belong to src). */
|
|
static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing,
|
|
const cbm_gbuf_node_t *sn, const gb_span_vec_t *sn_spans,
|
|
CBMHashTable **remap) {
|
|
/* Same guard as cbm_gbuf_upsert_node: a per-file "Module" def coming from a
|
|
* worker-local gbuf must not touch the structural directory node ("Project"
|
|
* root or "Folder") that shares its QN in a directory-based-module language
|
|
* (Java/Go). pass_structure seeds Folder/Project nodes on the MAIN gbuf
|
|
* before the parallel extract, so every worker's always-emitted Module def
|
|
* for that package collides here; unconditional "src wins" relabelled the
|
|
* directory node to Module and set its file_path to whichever worker merged
|
|
* LAST — the nondeterministic USAGE-source misattribution of #787. Keep the
|
|
* structural node intact; the ID remap below still redirects the worker's
|
|
* edges onto the canonical node. */
|
|
bool module_on_container =
|
|
existing->label && sn->label && strcmp(sn->label, "Module") == 0 &&
|
|
(strcmp(existing->label, "Project") == 0 || strcmp(existing->label, "Folder") == 0);
|
|
if (!module_on_container) {
|
|
/* Canonical collision winner (determinism) — mirrors
|
|
* cbm_gbuf_upsert_node exactly. Distinct source entities can share a
|
|
* QN (C: struct/function/macro with one name); unconditional "src
|
|
* wins" made the survivor depend on worker merge order, flickering
|
|
* the node set (and every downstream consumer) run to run. Winner =
|
|
* smallest file_path, then LARGEST start_line, then largest
|
|
* name/label — one total order, commutative, scheduling-free; a full
|
|
* tie is the same entity → refresh from src. A definition beats a
|
|
* macro ahead of all of these (gb_qn_claim_rank). */
|
|
int c = gb_qn_claim_rank(existing->label) - gb_qn_claim_rank(sn->label);
|
|
if (c == 0) {
|
|
c = strcmp(sn->file_path ? sn->file_path : "",
|
|
existing->file_path ? existing->file_path : "");
|
|
}
|
|
if (c == 0) {
|
|
c = existing->start_line - sn->start_line;
|
|
}
|
|
if (c == 0) {
|
|
c = strcmp(existing->name ? existing->name : "", sn->name ? sn->name : "");
|
|
}
|
|
if (c == 0) {
|
|
c = strcmp(existing->label ? existing->label : "", sn->label ? sn->label : "");
|
|
}
|
|
bool sn_wins = c <= 0;
|
|
gb_def_t src_def = {sn->label, sn->properties_json, sn->file_path, sn->start_line,
|
|
sn->end_line};
|
|
gb_note_collision(dst, existing, &src_def, sn_spans);
|
|
if (sn_wins) {
|
|
/* src's properties; the variant spans both carry are written on
|
|
* export (cbm_gbuf_settle_variants). */
|
|
/* Keep the secondary indexes consistent when the surviving
|
|
* label/name changes (the old code left the node listed under its
|
|
* original label/name, so find_by_label/name mis-listed it). */
|
|
const char *new_label = gb_intern(dst, sn->label);
|
|
bool label_changed =
|
|
!existing->label || !new_label || strcmp(existing->label, new_label) != 0;
|
|
bool name_changed =
|
|
!existing->name || !sn->name || strcmp(existing->name, sn->name) != 0;
|
|
if (label_changed) {
|
|
remove_node_from_ptr_array(
|
|
cbm_ht_get(dst->nodes_by_label, existing->label ? existing->label : ""),
|
|
existing->id);
|
|
}
|
|
if (name_changed) {
|
|
remove_node_from_ptr_array(
|
|
cbm_ht_get(dst->nodes_by_name, existing->name ? existing->name : ""),
|
|
existing->id);
|
|
}
|
|
existing->label = (char *)new_label;
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->name);
|
|
existing->name = heap_strdup(sn->name);
|
|
existing->file_path = (char *)gb_intern(dst, sn->file_path);
|
|
existing->start_line = sn->start_line;
|
|
existing->end_line = sn->end_line;
|
|
if (sn->properties_json) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json);
|
|
existing->properties_json = heap_strdup(sn->properties_json);
|
|
}
|
|
if (label_changed) {
|
|
node_ptr_array_t *by_label = get_or_create_node_array(
|
|
dst->nodes_by_label, existing->label ? existing->label : "");
|
|
node_array_push(by_label, existing);
|
|
}
|
|
if (name_changed) {
|
|
node_ptr_array_t *by_name = get_or_create_node_array(
|
|
dst->nodes_by_name, existing->name ? existing->name : "");
|
|
node_array_push(by_name, existing);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (sn->id != existing->id) {
|
|
if (!*remap) {
|
|
*remap = cbm_ht_create(CBM_SZ_32);
|
|
}
|
|
char key[CBM_SZ_32];
|
|
make_id_key(key, sizeof(key), sn->id);
|
|
int64_t *val = cbm_alloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(int64_t));
|
|
*val = existing->id;
|
|
cbm_ht_set(*remap, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), val);
|
|
}
|
|
}
|
|
|
|
/* Copy a non-colliding src node into dst with its original ID. */
|
|
static void merge_copy_new_node(cbm_gbuf_t *dst, const cbm_gbuf_node_t *sn) {
|
|
cbm_gbuf_node_t *node = cbm_calloc(CBM_MEM_CLASS_GBUF_NODE, sizeof(cbm_gbuf_node_t));
|
|
if (!node) {
|
|
return;
|
|
}
|
|
|
|
node->id = sn->id;
|
|
node->label = (char *)gb_intern(dst, sn->label);
|
|
node->name = heap_strdup(sn->name);
|
|
node->qualified_name = heap_strdup(sn->qualified_name);
|
|
node->file_path = (char *)gb_intern(dst, sn->file_path);
|
|
node->start_line = sn->start_line;
|
|
node->end_line = sn->end_line;
|
|
node->properties_json = heap_strdup(sn->properties_json);
|
|
|
|
cbm_da_push(&dst->nodes, node);
|
|
register_node_in_indexes(dst, node);
|
|
|
|
if (node->id >= dst->next_id) {
|
|
dst->next_id = node->id + SKIP_ONE;
|
|
}
|
|
}
|
|
|
|
/* Remap edge IDs using the collision remap table and insert into dst. */
|
|
static void merge_remap_edges(cbm_gbuf_t *dst, cbm_gbuf_t *src, CBMHashTable *remap) {
|
|
for (int i = 0; i < src->edges.count; i++) {
|
|
cbm_gbuf_edge_t *se = src->edges.items[i];
|
|
|
|
int64_t new_src = se->source_id;
|
|
int64_t new_tgt = se->target_id;
|
|
|
|
if (remap) {
|
|
char key[CBM_SZ_32];
|
|
make_id_key(key, sizeof(key), se->source_id);
|
|
int64_t *remapped = cbm_ht_get(remap, key);
|
|
if (remapped) {
|
|
new_src = *remapped;
|
|
}
|
|
|
|
make_id_key(key, sizeof(key), se->target_id);
|
|
remapped = cbm_ht_get(remap, key);
|
|
if (remapped) {
|
|
new_tgt = *remapped;
|
|
}
|
|
}
|
|
|
|
cbm_gbuf_insert_edge(dst, new_src, new_tgt, se->type, se->properties_json);
|
|
}
|
|
}
|
|
|
|
int cbm_gbuf_merge(cbm_gbuf_t *dst, cbm_gbuf_t *src) {
|
|
if (!dst || !src) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
if (src->captured_inputs_taken || dst->captured_inputs_taken) {
|
|
dst->resolution_failed = true;
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
if (src->resolution_failed)
|
|
dst->resolution_failed = true;
|
|
for (size_t i = 0; i < src->captured_input_count; i++) {
|
|
const cbm_gbuf_captured_input_t *input = &src->captured_inputs[i];
|
|
if (!cbm_gbuf_capture_input(dst, input->relative_path, input->present, input->byte_length,
|
|
input->content_identity))
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
if (!cbm_gbuf_resolution_begin(dst))
|
|
return CBM_NOT_FOUND;
|
|
bool resolver_ok = true;
|
|
for (int i = 0; resolver_ok && i < src->import_snapshots.count; i++) {
|
|
const cbm_resolve_import_snapshot_t *snapshot = src->import_snapshots.items[i];
|
|
if (cbm_gbuf_imports(src, snapshot->source.file) == snapshot)
|
|
resolver_ok = cbm_gbuf_capture_imports(dst, snapshot);
|
|
}
|
|
for (size_t i = 0; resolver_ok && i < src->resolution_count; i++)
|
|
resolver_ok = cbm_gbuf_record_resolution(dst, &src->resolutions[i]);
|
|
for (int i = 0; resolver_ok && i < src->resolver_files.count; i++)
|
|
resolver_ok = cbm_gbuf_record_resolver_file(dst, &src->resolver_files.items[i]);
|
|
for (int i = 0; resolver_ok && i < src->resolver_summaries.count; i++)
|
|
resolver_ok = cbm_gbuf_record_resolver_summary(dst, &src->resolver_summaries.items[i]);
|
|
cbm_gbuf_resolution_end(dst);
|
|
if (!resolver_ok)
|
|
return CBM_NOT_FOUND;
|
|
if (src->nodes.count == 0 && src->edges.count == 0) {
|
|
return 0;
|
|
}
|
|
|
|
/* ID remap for QN-colliding nodes: "src_id" → (int64_t*) dst_id.
|
|
* Only populated when a src node's QN already exists in dst. */
|
|
CBMHashTable *remap = NULL;
|
|
|
|
for (int i = 0; i < src->nodes.count; i++) {
|
|
cbm_gbuf_node_t *sn = src->nodes.items[i];
|
|
if (!sn->qualified_name) {
|
|
continue;
|
|
}
|
|
|
|
/* Skip nodes deleted from QN index */
|
|
if (!cbm_ht_get(src->node_by_qn, sn->qualified_name)) {
|
|
continue;
|
|
}
|
|
|
|
/* The spans src gathered for this node travel with it; they are
|
|
* written once, when dst is exported. */
|
|
const gb_span_vec_t *spans =
|
|
src->variant_spans ? cbm_ht_get(src->variant_spans, sn->qualified_name) : NULL;
|
|
cbm_gbuf_node_t *existing = cbm_ht_get(dst->node_by_qn, sn->qualified_name);
|
|
if (existing) {
|
|
merge_update_existing(dst, existing, sn, spans, &remap);
|
|
} else {
|
|
merge_copy_new_node(dst, sn);
|
|
gb_adopt_spans(dst, cbm_ht_get(dst->node_by_qn, sn->qualified_name), spans);
|
|
}
|
|
}
|
|
|
|
/* Merge edges with optional ID remapping */
|
|
merge_remap_edges(dst, src, remap);
|
|
|
|
if (remap) {
|
|
cbm_ht_foreach(remap, free_remap_entry, NULL);
|
|
cbm_ht_free(remap);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* ── Dump / Flush ────────────────────────────────────────────────── */
|
|
|
|
/* Extract a string property from a properties JSON string.
|
|
* Returns heap-allocated string or NULL. Caller must free.
|
|
* Parses real JSON: the dump writer feeds these values into indexes whose
|
|
* backing columns are GENERATED AS json_extract(properties,'$.<key>').
|
|
* Naive byte slicing returned the ESCAPED text (and cut at embedded \\")
|
|
* while json_extract yields the unescaped value — the mismatch left rows
|
|
* "missing from index idx_edges_url_path" under PRAGMA integrity_check.
|
|
* key_quoted ("\"key\"") is a fast pre-filter to skip the JSON parse. */
|
|
static char *extract_prop_string(const char *props, const char *key_quoted, const char *key) {
|
|
if (!props || !strstr(props, key_quoted)) {
|
|
return NULL;
|
|
}
|
|
yyjson_doc *doc = yyjson_read(props, strlen(props), 0);
|
|
if (!doc) {
|
|
return NULL;
|
|
}
|
|
char *out = NULL;
|
|
yyjson_val *v = yyjson_obj_get(yyjson_doc_get_root(doc), key);
|
|
if (v && yyjson_is_str(v)) {
|
|
const char *sv = yyjson_get_str(v);
|
|
out = cbm_mem_strdup(CBM_MEM_CLASS_DUMP, sv);
|
|
}
|
|
yyjson_doc_free(doc);
|
|
return out;
|
|
}
|
|
|
|
static char *extract_url_path(const char *props) {
|
|
return extract_prop_string(props, "\"url_path\"", "url_path");
|
|
}
|
|
|
|
/* local_name feeds the hand-built sqlite_autoindex_edges_1 — its backing
|
|
* column local_name_gen is GENERATED only for IMPORTS edges (#768). */
|
|
static char *extract_local_name(const char *props) {
|
|
return extract_prop_string(props, "\"local_name\"", "local_name");
|
|
}
|
|
|
|
/* Remap a temp edge ID to its final sequential ID, or 0 if out of range. */
|
|
static int64_t remap_id(const int64_t *temp_to_final, int64_t max_temp_id, int64_t temp_id) {
|
|
return (temp_id < max_temp_id) ? temp_to_final[temp_id] : 0;
|
|
}
|
|
|
|
/* Build dump-ready node array with sequential IDs. Populates temp_to_final mapping. */
|
|
static int cmp_dump_vectors_by_id(const void *a, const void *b) {
|
|
int64_t da = ((const CBMDumpVector *)a)->node_id;
|
|
int64_t db = ((const CBMDumpVector *)b)->node_id;
|
|
return (da > db) - (da < db);
|
|
}
|
|
|
|
static CBMDumpNode *build_dump_nodes(cbm_gbuf_t *gb, int live_count, int64_t *temp_to_final,
|
|
int64_t max_temp_id, int *out_count,
|
|
cbm_gbuf_node_t ***src_out) {
|
|
size_t cap = (size_t)(live_count > 0 ? live_count : SKIP_ONE);
|
|
CBMDumpNode *dump_nodes = cbm_alloc(CBM_MEM_CLASS_DUMP, cap * sizeof(CBMDumpNode));
|
|
/* Parallel gbuf-node pointers so a streamed partition can free its heavy
|
|
* properties_json after the rows are persisted. NULL on OOM disables the
|
|
* per-partition free (the dump still succeeds). */
|
|
cbm_gbuf_node_t **src = cbm_alloc(CBM_MEM_CLASS_DUMP, cap * sizeof(cbm_gbuf_node_t *));
|
|
int idx = 0;
|
|
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
if (!n->qualified_name || !cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
|
continue;
|
|
}
|
|
|
|
int64_t final_id = idx + SKIP_ONE; /* 1-based sequential */
|
|
if (n->id < max_temp_id) {
|
|
temp_to_final[n->id] = final_id;
|
|
}
|
|
|
|
const char *fp = n->file_path ? n->file_path : "";
|
|
const char *props = n->properties_json ? n->properties_json : "{}";
|
|
dump_nodes[idx] = (CBMDumpNode){
|
|
.id = final_id,
|
|
.project = gb->project,
|
|
.label = n->label,
|
|
.name = n->name,
|
|
.qualified_name = n->qualified_name,
|
|
.file_path = fp,
|
|
.start_line = n->start_line,
|
|
.end_line = n->end_line,
|
|
.properties = props,
|
|
};
|
|
if (src) {
|
|
src[idx] = n;
|
|
}
|
|
idx++;
|
|
}
|
|
|
|
*out_count = idx;
|
|
*src_out = src;
|
|
return dump_nodes;
|
|
}
|
|
|
|
/* Build dump-ready edge array with remapped IDs. Returns url_paths and
|
|
* local_names (heap string arrays owned by the caller) via out params. */
|
|
static CBMDumpEdge *build_dump_edges(cbm_gbuf_t *gb, const int64_t *temp_to_final,
|
|
int64_t max_temp_id, int *out_count, char ***out_url_paths,
|
|
char ***out_local_names) {
|
|
/* Count valid edges (both endpoints resolved) */
|
|
int valid_edges = 0;
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
if (remap_id(temp_to_final, max_temp_id, e->source_id) > 0 &&
|
|
remap_id(temp_to_final, max_temp_id, e->target_id) > 0) {
|
|
valid_edges++;
|
|
}
|
|
}
|
|
|
|
size_t edge_cap = (size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE);
|
|
CBMDumpEdge *dump_edges = cbm_alloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(CBMDumpEdge));
|
|
char **url_paths = cbm_calloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(char *));
|
|
char **local_names = cbm_calloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(char *));
|
|
int idx = 0;
|
|
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
int64_t src = remap_id(temp_to_final, max_temp_id, e->source_id);
|
|
int64_t tgt = remap_id(temp_to_final, max_temp_id, e->target_id);
|
|
if (src == 0 || tgt == 0) {
|
|
continue;
|
|
}
|
|
|
|
char *url_path = extract_url_path(e->properties_json);
|
|
url_paths[idx] = url_path;
|
|
|
|
/* IMPORTS only — mirrors the local_name_gen CASE in the edges DDL. */
|
|
char *local_name = (e->type && strcmp(e->type, "IMPORTS") == 0)
|
|
? extract_local_name(e->properties_json)
|
|
: NULL;
|
|
local_names[idx] = local_name;
|
|
|
|
const char *props = e->properties_json ? e->properties_json : "{}";
|
|
dump_edges[idx] = (CBMDumpEdge){
|
|
.id = idx + SKIP_ONE,
|
|
.project = gb->project,
|
|
.source_id = src,
|
|
.target_id = tgt,
|
|
.type = e->type,
|
|
.properties = props,
|
|
.url_path = url_path ? url_path : "",
|
|
.local_name = local_name ? local_name : "",
|
|
};
|
|
idx++;
|
|
}
|
|
|
|
*out_count = idx;
|
|
*out_url_paths = url_paths;
|
|
*out_local_names = local_names;
|
|
return dump_edges;
|
|
}
|
|
|
|
/* Remap vector node IDs through temp_to_final, sort by ID, deduplicate. */
|
|
static void remap_sort_dedup_vectors(cbm_gbuf_t *gb, const int64_t *temp_to_final,
|
|
int64_t max_temp_id) {
|
|
int remapped = 0;
|
|
int dropped = 0;
|
|
for (int i = 0; i < gb->dump_vector_count; i++) {
|
|
int64_t old_id = gb->dump_vectors[i].node_id;
|
|
int64_t new_id = (old_id > 0 && old_id < max_temp_id) ? temp_to_final[old_id] : 0;
|
|
if (new_id > 0) {
|
|
gb->dump_vectors[remapped] = gb->dump_vectors[i];
|
|
gb->dump_vectors[remapped].node_id = new_id;
|
|
remapped++;
|
|
} else {
|
|
dropped++;
|
|
}
|
|
}
|
|
if (dropped > 0) {
|
|
char r_buf[CBM_SZ_16];
|
|
char d_buf[CBM_SZ_16];
|
|
snprintf(r_buf, sizeof(r_buf), "%d", remapped);
|
|
snprintf(d_buf, sizeof(d_buf), "%d", dropped);
|
|
cbm_log_info("dump.vectors.remap", "remapped", r_buf, "dropped", d_buf);
|
|
}
|
|
gb->dump_vector_count = remapped;
|
|
|
|
if (gb->dump_vector_count >= GB_MIN_FOR_DEDUP) {
|
|
qsort(gb->dump_vectors, (size_t)gb->dump_vector_count, sizeof(CBMDumpVector),
|
|
cmp_dump_vectors_by_id);
|
|
int deduped = 0;
|
|
for (int i = 0; i < gb->dump_vector_count; i++) {
|
|
if (i + GB_DEDUP_LOOKAHEAD < gb->dump_vector_count &&
|
|
gb->dump_vectors[i].node_id == gb->dump_vectors[i + GB_DEDUP_LOOKAHEAD].node_id) {
|
|
continue;
|
|
}
|
|
gb->dump_vectors[deduped++] = gb->dump_vectors[i];
|
|
}
|
|
gb->dump_vector_count = deduped;
|
|
}
|
|
}
|
|
|
|
static void log_dump_summary(int node_count, int edge_count) {
|
|
char b1[CBM_SZ_16];
|
|
char b2[CBM_SZ_16];
|
|
snprintf(b1, sizeof(b1), "%d", node_count);
|
|
snprintf(b2, sizeof(b2), "%d", edge_count);
|
|
cbm_log_info("gbuf.dump", "nodes", b1, "edges", b2);
|
|
}
|
|
|
|
/* A dump that published nothing gets exactly one signature in the log.
|
|
* Without it the writer's return code is the only evidence, and it reaches
|
|
* the user as a generic pipeline failure that blames their repository
|
|
* (#1620, #2001).
|
|
*
|
|
* failure_errno is read by the caller at the point of failure: the value is
|
|
* only meaningful before the next library call. Zero means the failure did
|
|
* not originate at the publish boundary — an append that failed many calls
|
|
* earlier — and the field is omitted rather than claim a reason this record
|
|
* does not have. */
|
|
static void log_dump_failed(int rc, int failure_errno, int node_count, int edge_count) {
|
|
char b1[CBM_SZ_16];
|
|
char b3[CBM_SZ_16];
|
|
char b4[CBM_SZ_16];
|
|
snprintf(b1, sizeof(b1), "%d", rc);
|
|
snprintf(b3, sizeof(b3), "%d", node_count);
|
|
snprintf(b4, sizeof(b4), "%d", edge_count);
|
|
if (failure_errno == 0) {
|
|
cbm_log_error("gbuf.dump_failed", "rc", b1, "nodes", b3, "edges", b4);
|
|
return;
|
|
}
|
|
char b2[CBM_SZ_16];
|
|
snprintf(b2, sizeof(b2), "%d", failure_errno);
|
|
cbm_log_error("gbuf.dump_failed", "rc", b1, "errno", b2, "nodes", b3, "edges", b4);
|
|
}
|
|
|
|
static void free_dump_resources(char **url_paths, char **local_names, int edge_count,
|
|
CBMDumpEdge *dump_edges, CBMDumpNode *dump_nodes,
|
|
int64_t *temp_to_final) {
|
|
for (int i = 0; i < edge_count; i++) {
|
|
cbm_free(CBM_MEM_CLASS_DUMP, url_paths[i]);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, local_names[i]);
|
|
}
|
|
cbm_free(CBM_MEM_CLASS_DUMP, url_paths);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, local_names);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, dump_edges);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final);
|
|
}
|
|
|
|
static int count_live_nodes(cbm_gbuf_t *gb) {
|
|
int count = 0;
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
if (n->qualified_name && cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
|
count++;
|
|
}
|
|
}
|
|
return count;
|
|
}
|
|
|
|
static void generate_iso_timestamp(char *buf, size_t buf_size) {
|
|
time_t now = time(NULL);
|
|
struct tm tm_buf;
|
|
struct tm *tm_val = cbm_gmtime_r(&now, &tm_buf);
|
|
if (strftime(buf, buf_size, "%Y-%m-%dT%H:%M:%SZ", tm_val) == 0) {
|
|
snprintf(buf, buf_size, "1970-01-01T00:00:00Z");
|
|
}
|
|
}
|
|
|
|
/* Release lookup indexes then remap+sort+dedup vectors for the B-tree writer. */
|
|
static void release_and_remap_vectors(cbm_gbuf_t *gb, const int64_t *temp_to_final,
|
|
int64_t max_temp_id) {
|
|
CBM_PROF_START(t_vec_remap);
|
|
remap_sort_dedup_vectors(gb, temp_to_final, max_temp_id);
|
|
CBM_PROF_END_N("dump", "5_vector_remap_sort", t_vec_remap, gb->dump_vector_count);
|
|
}
|
|
|
|
int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
|
|
if (!gb || !path) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
cbm_gbuf_settle_variants(gb);
|
|
|
|
CBM_PROF_START(t_count);
|
|
int live_count = count_live_nodes(gb);
|
|
CBM_PROF_END_N("dump", "1_count_live_nodes", t_count, live_count);
|
|
|
|
CBM_PROF_START(t_build_nodes);
|
|
int64_t max_temp_id = gb->next_id;
|
|
int64_t *temp_to_final = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t));
|
|
if (!temp_to_final) {
|
|
log_dump_failed(CBM_NOT_FOUND, errno, 0, 0);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
int node_idx = 0;
|
|
cbm_gbuf_node_t **src_nodes = NULL;
|
|
CBMDumpNode *dump_nodes =
|
|
build_dump_nodes(gb, live_count, temp_to_final, max_temp_id, &node_idx, &src_nodes);
|
|
CBM_PROF_END_N("dump", "2_build_dump_nodes", t_build_nodes, node_idx);
|
|
|
|
/* Release ALL lookup indexes NOW: nothing between here and finalize reads
|
|
* them (stream-append uses dump_nodes/src_nodes, build_dump_edges uses
|
|
* temp_to_final + gb->edges, the vector remap uses temp_to_final). At
|
|
* kernel scale they are ~3.8 GB (hash buckets + key strings + pointer
|
|
* arrays); releasing them only AFTER edge building made them coexist with
|
|
* every dump-side transient array — the dump-phase RSS peak. */
|
|
CBM_PROF_START(t_release_idx);
|
|
release_gbuf_indexes(gb);
|
|
CBM_PROF_END("dump", "4_release_gbuf_indexes", t_release_idx);
|
|
|
|
char indexed_at[CBM_SZ_64];
|
|
generate_iso_timestamp(indexed_at, sizeof(indexed_at));
|
|
|
|
/* Stream node rows to the DB in partitions. Under memory pressure, free each
|
|
* partition's heavy properties_json once persisted — the heavy column is
|
|
* write-once and never read again, so this bounds the dump/finalize peak.
|
|
* The DB output is identical whether or not freeing engages, so non-pressure
|
|
* runs (and tests) leave the gbuf intact (the budget>0 guard keeps an
|
|
* uninitialized budget from ever triggering the free). */
|
|
cbm_db_writer_t *w = cbm_writer_open(path);
|
|
if (!w) {
|
|
log_dump_failed(CBM_NOT_FOUND, errno, node_idx, 0);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, src_nodes);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final);
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
|
|
CBM_PROF_START(t_append);
|
|
enum { DUMP_PARTITION_NODES = 1 << 16 };
|
|
bool free_heavy = false;
|
|
int rc = 0;
|
|
for (int off = 0; off < node_idx; off += DUMP_PARTITION_NODES) {
|
|
int chunk = node_idx - off;
|
|
if (chunk > DUMP_PARTITION_NODES) {
|
|
chunk = DUMP_PARTITION_NODES;
|
|
}
|
|
rc = cbm_writer_append_nodes(w, &dump_nodes[off], chunk);
|
|
if (rc != 0) {
|
|
break;
|
|
}
|
|
/* cbm_mem_should_relieve, not cbm_mem_over_budget: the dump is where
|
|
* the run died on 2026-09-18 with our charge UNDER budget and the host
|
|
* out of memory, so the only signal that would have fired here is the
|
|
* machine's. Dropping the property strings we have already written is
|
|
* pure relief — the rows are in the file by now. */
|
|
free_heavy = free_heavy || (cbm_mem_budget() > 0 && cbm_mem_should_relieve());
|
|
if (free_heavy && src_nodes) {
|
|
for (int j = off; j < off + chunk; j++) {
|
|
cbm_free(CBM_MEM_CLASS_GBUF_STRING, src_nodes[j]->properties_json);
|
|
src_nodes[j]->properties_json = NULL;
|
|
dump_nodes[j].properties = NULL;
|
|
}
|
|
cbm_mem_collect();
|
|
}
|
|
}
|
|
CBM_PROF_END_N("dump", "2b_stream_append_nodes", t_append, node_idx);
|
|
|
|
int edge_idx = 0;
|
|
char **url_paths = NULL;
|
|
char **local_names = NULL;
|
|
CBMDumpEdge *dump_edges = NULL;
|
|
if (rc == 0) {
|
|
CBM_PROF_START(t_build_edges);
|
|
dump_edges =
|
|
build_dump_edges(gb, temp_to_final, max_temp_id, &edge_idx, &url_paths, &local_names);
|
|
CBM_PROF_END_N("dump", "3_build_dump_edges", t_build_edges, edge_idx);
|
|
release_and_remap_vectors(gb, temp_to_final, max_temp_id);
|
|
}
|
|
|
|
/* Finalize: nodes-table interior + edges/vectors/metadata/indexes/sqlite_master.
|
|
* Frees w and closes the file; handles a prior append error cleanly. */
|
|
CBM_PROF_START(t_finalize);
|
|
int frc = cbm_writer_finalize(w, gb->project, gb->root_path, indexed_at, dump_nodes, node_idx,
|
|
dump_edges, edge_idx, gb->dump_vectors, gb->dump_vector_count,
|
|
gb->dump_token_vecs, gb->dump_token_vec_count);
|
|
/* Read errno before anything else can touch it: the profiling macro
|
|
* below is already one library call away from losing the reason. */
|
|
int finalize_errno = errno;
|
|
CBM_PROF_END_N("dump", "6_write_db_finalize", t_finalize, node_idx + edge_idx);
|
|
if (rc == 0) {
|
|
rc = frc;
|
|
}
|
|
|
|
if (rc != 0) {
|
|
log_dump_failed(rc, finalize_errno, node_idx, edge_idx);
|
|
} else {
|
|
log_dump_summary(node_idx, edge_idx);
|
|
}
|
|
free_dump_resources(url_paths, local_names, edge_idx, dump_edges, dump_nodes, temp_to_final);
|
|
cbm_free(CBM_MEM_CLASS_DUMP, src_nodes);
|
|
return rc;
|
|
}
|
|
|
|
int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) {
|
|
if (!gb || !store) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
cbm_gbuf_settle_variants(gb);
|
|
|
|
/* Upsert project */
|
|
cbm_store_upsert_project(store, gb->project, gb->root_path);
|
|
|
|
/* Begin bulk mode */
|
|
cbm_store_begin_bulk(store);
|
|
cbm_store_drop_indexes(store);
|
|
cbm_store_begin(store);
|
|
|
|
/* Delete existing project data */
|
|
cbm_store_delete_edges_by_project(store, gb->project);
|
|
cbm_store_delete_nodes_by_project(store, gb->project);
|
|
|
|
/* Build temp_id → real_id map.
|
|
* Temp IDs start at 1 and are sequential, but can have gaps from edge inserts.
|
|
* Use max_id as size. */
|
|
int64_t max_temp_id = gb->next_id;
|
|
int64_t *temp_to_real = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t));
|
|
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
|
|
/* Skip if deleted from QN index */
|
|
if (!n->qualified_name || !cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
|
continue;
|
|
}
|
|
|
|
cbm_node_t sn = {
|
|
.project = gb->project,
|
|
.label = n->label,
|
|
.name = n->name,
|
|
.qualified_name = n->qualified_name,
|
|
.file_path = n->file_path,
|
|
.start_line = n->start_line,
|
|
.end_line = n->end_line,
|
|
.properties_json = n->properties_json,
|
|
};
|
|
int64_t real_id = cbm_store_upsert_node(store, &sn);
|
|
if (real_id > 0 && n->id < max_temp_id) {
|
|
temp_to_real[n->id] = real_id;
|
|
}
|
|
}
|
|
|
|
/* Insert all edges with remapped IDs */
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
int64_t real_src = (e->source_id < max_temp_id) ? temp_to_real[e->source_id] : 0;
|
|
int64_t real_tgt = (e->target_id < max_temp_id) ? temp_to_real[e->target_id] : 0;
|
|
if (real_src == 0 || real_tgt == 0) {
|
|
continue;
|
|
}
|
|
|
|
cbm_edge_t se = {
|
|
.project = gb->project,
|
|
.source_id = real_src,
|
|
.target_id = real_tgt,
|
|
.type = e->type,
|
|
.properties_json = e->properties_json,
|
|
};
|
|
cbm_store_insert_edge(store, &se);
|
|
}
|
|
|
|
cbm_store_commit(store);
|
|
cbm_store_create_indexes(store);
|
|
cbm_store_end_bulk(store);
|
|
|
|
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_real);
|
|
return 0;
|
|
}
|
|
|
|
int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) {
|
|
if (!gb || !store) {
|
|
return CBM_NOT_FOUND;
|
|
}
|
|
cbm_gbuf_settle_variants(gb);
|
|
|
|
/* Begin bulk mode — no project wipe */
|
|
cbm_store_begin(store);
|
|
|
|
/* Build temp_id → real_id map */
|
|
int64_t max_temp_id = gb->next_id;
|
|
int64_t *temp_to_real = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t));
|
|
|
|
for (int i = 0; i < gb->nodes.count; i++) {
|
|
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
|
|
|
if (!n->qualified_name || !cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
|
continue;
|
|
}
|
|
|
|
cbm_node_t sn = {
|
|
.project = gb->project,
|
|
.label = n->label,
|
|
.name = n->name,
|
|
.qualified_name = n->qualified_name,
|
|
.file_path = n->file_path,
|
|
.start_line = n->start_line,
|
|
.end_line = n->end_line,
|
|
.properties_json = n->properties_json,
|
|
};
|
|
int64_t real_id = cbm_store_upsert_node(store, &sn);
|
|
if (real_id > 0 && n->id < max_temp_id) {
|
|
temp_to_real[n->id] = real_id;
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < gb->edges.count; i++) {
|
|
cbm_gbuf_edge_t *e = gb->edges.items[i];
|
|
int64_t real_src = (e->source_id < max_temp_id) ? temp_to_real[e->source_id] : 0;
|
|
int64_t real_tgt = (e->target_id < max_temp_id) ? temp_to_real[e->target_id] : 0;
|
|
if (real_src == 0 || real_tgt == 0) {
|
|
continue;
|
|
}
|
|
|
|
cbm_edge_t se = {
|
|
.project = gb->project,
|
|
.source_id = real_src,
|
|
.target_id = real_tgt,
|
|
.type = e->type,
|
|
.properties_json = e->properties_json,
|
|
};
|
|
cbm_store_insert_edge(store, &se);
|
|
}
|
|
|
|
cbm_store_commit(store);
|
|
|
|
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_real);
|
|
return 0;
|
|
}
|