From 11cb476dff14194ba772f3dda6b9be640e3f6e0e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 19 Sep 2026 15:50:11 +0200 Subject: [PATCH] fix(semantic): inject callee tokens in a fixed order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit inject_calls_pattern_tokens read a node's CALLS edges straight out of the graph buffer, which holds them in insertion order — and insertion order is the order parallel workers happened to merge. It then injected the callee names in that order and truncated at max_tokens, so which callees survived depended on worker scheduling. Co-occurrence enrichment reads a window around each token position, so the surviving vectors differed between runs while the document frequencies (and therefore idf) stayed identical, which is why this showed up as a handful of SEMANTICALLY_RELATED edges flickering and nothing else. It now collects the neighbours through collect_sorted_call_neighbors(), the same canonical ordering the rest of the pass already uses. Three runs of the Go corpus now agree exactly — 334535 nodes / 1795409 edges with vec=0e45744ff1042b51 idf=7e994a1d864a7b07 toks=2518a08a0b6af0c7 — and so do php, kotlin and rust. The edge counts move slightly against the previous build (php 57182 -> 57198, kotlin +17, rust +3): sorting changes which tokens survive the truncation, so the semantic edges change with it. That is a deliberate output change, not a regression. Also adds the checksums that found it, all off unless their env var is set: CBM_SEM_STATE_DUMP writes one hash per scoring input (codes, tf-idf, struct profile, minhash) and one per corpus (ri vectors, idf, and the token SEQUENCE, which is what differed here), and CBM_SEM_PAIR_DUMP writes every candidate pair that passed the threshold with its score, in canonical order. This subsystem has now produced three order-dependence bugs; a checksum at the seam is what turns "the graph differs" into a localised answer in minutes, and it costs nothing when the variable is unset. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 188 +++++++++++++++++++++++++++-- tests/test_vm_verdict_contract.sh | 72 +++++++++++ 2 files changed, 251 insertions(+), 9 deletions(-) create mode 100755 tests/test_vm_verdict_contract.sh diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index dd8057d61..05a7387a9 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -22,6 +22,7 @@ #include "foundation/hash_table.h" #include "foundation/log.h" #include "foundation/compat.h" +#include "foundation/compat_fs.h" /* cbm_fopen for the pair dump */ #define XXH_INLINE_ALL #include "xxhash/xxhash.h" @@ -194,23 +195,33 @@ static int inject_callee_tokens(const char *name, char **tokens, int count, int } /* Walk the outbound CALLS edges of n and inject tokens for each target. */ +static const char **collect_sorted_call_neighbors(const cbm_gbuf_t *gbuf, int64_t node_id, + bool outbound, int *out_n); + static int inject_calls_pattern_tokens(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, char **tokens, int count, int max_tokens) { if (!gbuf) { return count; } - const cbm_gbuf_edge_t **edges = NULL; - int ec = 0; - if (cbm_gbuf_find_edges_by_source_type(gbuf, n->id, "CALLS", &edges, &ec) != 0) { + /* SORTED, like every other consumer of CALLS neighbours in this file. The + * raw edge array is in graph-buffer insertion order — parallel worker merge + * order — and this loop both ORDERS the injected tokens and truncates at + * max_tokens. Co-occurrence enrichment reads a window around each token + * POSITION, so a different order yields different token vectors while + * leaving document frequencies untouched: measured on the go corpus + * 2026-09-19, two runs of one binary with identical idf but different token + * sequences and different enriched vectors, which flickered near-threshold + * SEMANTICALLY_RELATED edges. collect_sorted_call_neighbors() was written + * for exactly this class and this call site was missed. */ + int nn = 0; + const char **names = collect_sorted_call_neighbors(gbuf, n->id, /*outbound=*/true, &nn); + if (!names) { return count; } - for (int e = 0; e < ec && count < max_tokens; e++) { - const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gbuf, edges[e]->target_id); - if (!t || !t->name) { - continue; - } - count = inject_callee_tokens(t->name, tokens, count, max_tokens); + for (int e = 0; e < nn && count < max_tokens; e++) { + count = inject_callee_tokens(names[e], tokens, count, max_tokens); } + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)names); return count; } @@ -1152,9 +1163,27 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf } qsort(pairs, (size_t)n, sizeof(deferred_edge_t), cmp_deferred_edge_canonical); + /* CBM_SEM_PAIR_DUMP=: every candidate that passed the threshold, with + * its score, in canonical order. Written HERE because this loop is + * single-threaded and already sorted, so the dump needs no lock and two + * runs compare line by line. The (i, j) indices are stable across runs — + * funcs[] is sorted by qualified name — where node ids are not. Exists to + * chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that + * flips, then compare its score between runs. */ + FILE *pair_dump = NULL; + { + char dump_path[CBM_SZ_1K]; + if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) { + pair_dump = cbm_fopen(dump_path, "w"); + } + } + int total_edges = 0; for (int e = 0; e < n; e++) { deferred_edge_t *de = &pairs[e]; + if (pair_dump) { + fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score); + } if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) { continue; } @@ -1166,6 +1195,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf edge_counts[de->j]++; total_edges++; } + if (pair_dump) { + (void)fclose(pair_dump); + } cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs); return total_edges; } @@ -1370,6 +1402,139 @@ static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_c *out_buckets = band_buckets; } +/* Checksum every input the scorer reads, so two runs can be compared at the + * seam right before scoring. cbm_sem_combined_score() reads exactly these + * fields, so identical checksums with differing scores would be impossible. + * Placed AFTER tf-idf is filled (phase 4), not after phase 1b where the + * sparse vectors are still empty. Diagnostic, off unless the env var is set. */ +static void sem_state_dump(const cbm_sem_func_t *funcs, int func_count) { + /* CBM_SEM_STATE_DUMP=1: one checksum over every per-function vector, so two + * runs can be compared at THIS seam. Scores are pure math over these + * vectors, so if the checksums match and the scores do not, the difference + * is downstream; if they differ, it is here or above. Diagnostic only. */ + char state_path[CBM_SZ_1K]; + if (cbm_safe_getenv("CBM_SEM_STATE_DUMP", state_path, sizeof(state_path), NULL)) { + uint64_t h = 1469598103934665603ULL; + uint64_t h_codes = 1469598103934665603ULL; + uint64_t h_tfidf = 1469598103934665603ULL; + for (int i = 0; i < func_count; i++) { + const unsigned char *bytes[] = { + (const unsigned char *)&funcs[i].ri_code, + (const unsigned char *)&funcs[i].api_code, + (const unsigned char *)&funcs[i].type_code, + (const unsigned char *)&funcs[i].deco_code, + }; + for (size_t b = 0; b < sizeof(bytes) / sizeof(bytes[0]); b++) { + for (size_t k = 0; k < sizeof(cbm_rsq_code_t); k++) { + h ^= bytes[b][k]; + h *= 1099511628211ULL; + h_codes ^= bytes[b][k]; + h_codes *= 1099511628211ULL; + } + } + /* The other scoring inputs: sparse TF-IDF, the AST profile and the + * minhash fingerprint. Scoring reads exactly these plus the codes + * above, so a matching checksum means the scores must match too. */ + const unsigned char *sp = (const unsigned char *)funcs[i].struct_profile; + for (size_t k = 0; k < sizeof(funcs[i].struct_profile); k++) { + h ^= sp[k]; + h *= 1099511628211ULL; + } + const unsigned char *mh = (const unsigned char *)funcs[i].minhash; + for (size_t k = 0; k < (funcs[i].has_minhash ? sizeof(funcs[i].minhash) : 0); k++) { + h ^= mh[k]; + h *= 1099511628211ULL; + } + for (int w = 0; w < funcs[i].tfidf_len; w++) { + h ^= (uint64_t)funcs[i].tfidf_indices[w]; + h *= 1099511628211ULL; + uint32_t bits; + memcpy(&bits, &funcs[i].tfidf_weights[w], sizeof(bits)); + h ^= bits; + h *= 1099511628211ULL; + h_tfidf ^= (uint64_t)funcs[i].tfidf_indices[w]; + h_tfidf *= 1099511628211ULL; + h_tfidf ^= bits; + h_tfidf *= 1099511628211ULL; + } + } + /* To a FILE, not the log: the semantic pass runs in the index WORKER, + * whose log is removed when the run succeeds. */ + FILE *sf = cbm_fopen(state_path, "w"); + if (sf) { + fprintf(sf, "all=%016llx codes=%016llx tfidf=%016llx n=%d\n", (unsigned long long)h, + (unsigned long long)h_codes, (unsigned long long)h_tfidf, func_count); + (void)fclose(sf); + } + } +} + +/* Running hash of the token SEQUENCE handed to the corpus: the set AND the + * order, because co-occurrence enrichment reads a window around each position, + * so the same tokens in a different order produce different vectors while + * leaving document frequencies (and therefore idf) untouched. Diagnostic for + * the SEMANTICALLY_RELATED flicker; off unless CBM_SEM_STATE_DUMP is set. */ +static uint64_t g_sem_token_hash = 1469598103934665603ULL; + +static void sem_tokens_hash(char **tokens, const size_t *offsets, const int *counts, int n) { + char flag[CBM_SZ_1K]; + if (!cbm_safe_getenv("CBM_SEM_STATE_DUMP", flag, sizeof(flag), NULL)) { + return; + } + for (int d = 0; d < n; d++) { + for (int k = 0; k < counts[d]; k++) { + const char *tok = tokens[offsets[d] + (size_t)k]; + for (const unsigned char *c = (const unsigned char *)tok; c && *c; c++) { + g_sem_token_hash ^= *c; + g_sem_token_hash *= 1099511628211ULL; + } + g_sem_token_hash ^= 0xff; /* token boundary */ + g_sem_token_hash *= 1099511628211ULL; + } + g_sem_token_hash ^= 0xfe; /* document boundary */ + g_sem_token_hash *= 1099511628211ULL; + } +} + +/* Checksum the corpus AFTER enrichment: the per-function vectors and tf-idf are + * both derived from it, so if this differs between two runs, everything + * downstream differs and the cause is in the corpus build rather than in + * scoring. Diagnostic, off unless CBM_SEM_STATE_DUMP is set. */ +static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix) { + char state_path[CBM_SZ_1K]; + if (!cbm_safe_getenv("CBM_SEM_STATE_DUMP", state_path, sizeof(state_path), NULL)) { + return; + } + char path[CBM_SZ_1K]; + if (snprintf(path, sizeof(path), "%s.%s", state_path, suffix) >= (int)sizeof(path)) { + return; + } + int n = cbm_sem_corpus_token_count(corpus); + uint64_t h_vec = 1469598103934665603ULL; + uint64_t h_idf = 1469598103934665603ULL; + for (int i = 0; i < n; i++) { + const cbm_sem_vec_t *v = cbm_sem_corpus_ri_vec_at(corpus, i); + if (v) { + const unsigned char *b = (const unsigned char *)v; + for (size_t k = 0; k < sizeof(*v); k++) { + h_vec ^= b[k]; + h_vec *= 1099511628211ULL; + } + } + uint32_t bits; + float idf = cbm_sem_corpus_idf_at(corpus, i); + memcpy(&bits, &idf, sizeof(bits)); + h_idf ^= bits; + h_idf *= 1099511628211ULL; + } + FILE *sf = cbm_fopen(path, "w"); + if (sf) { + fprintf(sf, "vec=%016llx idf=%016llx toks=%016llx tokens=%d\n", (unsigned long long)h_vec, + (unsigned long long)h_idf, (unsigned long long)g_sem_token_hash, n); + (void)fclose(sf); + } +} + /* Phase 6a: score candidate pairs in parallel and collect deferred edges. */ static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts, sem_bucket_t **band_buckets, cbm_sem_config_t cfg, @@ -1415,11 +1580,13 @@ static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, int func_count) { CBM_PROF_START(t_phase3a); cbm_sem_corpus_t *corpus = cbm_sem_corpus_new(); + sem_tokens_hash(all_tokens, offsets, token_counts, func_count); cbm_sem_corpus_add_docs_batch(corpus, all_tokens, offsets, token_counts, func_count); CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count); CBM_PROF_START(t_phase3b); cbm_sem_corpus_finalize(corpus); + sem_corpus_dump(corpus, "corpus"); CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b, cbm_sem_corpus_token_count(corpus)); @@ -1449,6 +1616,7 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t * } CBM_PROF_START(t_phase6a); + sem_state_dump(funcs, func_count); phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, func_count, worker_count); CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); @@ -1599,6 +1767,7 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func sem_batch_tokenize(node_ptrs, gbuf, first, count, worker_count, token_pools, token_counts, &tokens, &offsets); if (tokens && offsets) { + sem_tokens_hash(tokens, offsets, token_counts + first, count); cbm_sem_corpus_add_docs_batch(corpus, tokens, offsets, token_counts + first, count); } sem_batch_free_tokens(tokens, offsets); @@ -1609,6 +1778,7 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func CBM_PROF_START(t_phase3b); cbm_sem_corpus_finalize(corpus); + sem_corpus_dump(corpus, "corpus"); CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b, cbm_sem_corpus_token_count(corpus)); CBM_PROF_START(t_phase3c); diff --git a/tests/test_vm_verdict_contract.sh b/tests/test_vm_verdict_contract.sh new file mode 100755 index 000000000..3076eb820 --- /dev/null +++ b/tests/test_vm_verdict_contract.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# The Windows leg's verdict must not be decided by the exit status alone. +# +# Why: that status travels ssh -> cmd.exe -> msys2_shell.cmd and the chain +# loses it. On 2026-09-18 a leg that printed "7925 passed, 0 failed" and +# "=== All tests passed ===" exited 1, while the identical work run through +# win.sh's other entry exited 0. A channel that turns 0 into 1 can turn 1 into +# 0, and that direction is a FALSE GREEN: a red Windows leg reported as +# passing, which is exactly the class of defect the local ladder exists to +# catch. The log is written by the runner on the VM and cannot be mangled in +# transit, so it decides the test outcome; rc only decides what the log cannot +# see. +# +# This drives the REAL function (vm-run-tests.sh --verdict) rather than a copy, +# so the contract cannot drift from the code it pins. +# +# Usage: tests/test_vm_verdict_contract.sh [repo-root] + +set -uo pipefail + +ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}" +RUNNER="$ROOT/test-infrastructure/vm/vm-run-tests.sh" +if [ ! -f "$RUNNER" ]; then + echo "FAIL: $RUNNER not found" >&2 + exit 1 +fi + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/cbm-vm-verdict.XXXXXX") || exit 1 +trap 'rm -rf "$WORK"' EXIT + +printf ' 7925 passed, 0 failed, 74 skipped\n=== All tests passed ===\n' > "$WORK/green" +printf ' 7863 passed, 1 failed, 74 skipped\n' > "$WORK/red" +printf ' 7863 passed, 1 failed\n=== All tests passed ===\n' > "$WORK/red_with_marker" +printf ' 100 passed, 0 failed\n 200 passed, 3 failed\n=== All tests passed ===\n' > "$WORK/red_later_suite" +printf ' 7925 passed, 0 failed, 74 skipped\n' > "$WORK/no_marker" +printf 'configure: something exploded\n' > "$WORK/no_summary" + +failures=0 +expect() { # description log rc want + local description="$1" log="$2" rc="$3" want="$4" got + bash "$RUNNER" --verdict "$WORK/$log" "$rc" >/dev/null 2>&1 + got=$? + if [ "$got" -ne "$want" ]; then + echo "FAIL: $description — verdict $got, expected $want" >&2 + failures=$((failures + 1)) + fi +} + +# The observed fault: a complete, green log whose exit status was lost. The +# leg is green; the transport problem is reported, not the tests. +expect "green log with a lost exit status is green" green 1 0 +expect "green log with a clean exit status is green" green 0 0 + +# The dangerous direction. A zero exit status must never launder failures. +expect "failures in the log are red even when rc says 0" red 0 1 +expect "failures in the log are red when rc says 1" red 1 1 +expect "a completion marker cannot launder failures" red_with_marker 0 1 +expect "failures in a LATER suite are still counted" red_later_suite 0 1 + +# Incomplete runs are never green: a leg that stopped after one suite has a +# summary but no marker. +expect "a summary without the completion marker is red" no_marker 0 1 + +# No summary at all keeps its own distinct code, so "never ran" stays +# distinguishable from "ran and failed". +expect "a log with no summary is the no-summary guard" no_summary 0 90 + +if [ "$failures" -gt 0 ]; then + echo "VM verdict contract VIOLATED: $failures case(s)" >&2 + exit 1 +fi +echo "VM verdict contract passed (8 cases: lost exit status, false green, partial runs)"