From c43b319ffc2a6ee10706c668d140212c9cc11fb0 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 13 Oct 2026 17:50:16 +0200 Subject: [PATCH] fix(extract): one definition per site when the raw and preprocessed parses disagree The #961/#1989 rescue re-adopts definitions from the preprocessed second pass when they sit in a raw parse error, and drops a copy only when its QN equals a raw one. When the two parses scope the same definition differently, both copies survived: - Linux lib/tests/randstruct_kunit.c: a macro-generated struct body makes the raw parse nest every later struct inside one ERROR region; the flat expanded copies were adopted next to the mis-nested raw ones (5 duplicate groups). - coreclr/rapidjson: the expanded parse nests utilcode.h classes wrongly or loses rapidjson's scopes, and both copies survived. One rule at one decision point: an expanded and a raw definition with the same label, name and start line but different QNs are one definition. The raw copy wins and whatever the expanded pass attributed to the expanded QN moves to it -- unless - the raw copy's nesting exists only because of a raw ERROR region that the expanded parse resolved (the randstruct shape, six conditions in cbm_misnested_raw_twin): then the flat expanded copy replaces the raw one and the raw children (Fields, nested defs, call/usage scopes) are re-parented to it; or - the raw copy's QN is already held by a raw definition of another label (coreclr inc/holder.h: the raw parse reads a FORCEINLINE constructor as a Function of the class's scope-less QN): then the expanded copy keeps its own, correct QN instead of vanishing into the Function's node. Proof (macOS arm64): - randstruct_kunit.c: 5 -> 0 duplicate groups, 12 Fields re-parented to the flat structs, CALLS/USAGE/WRITES unchanged. - dotnet/runtime native subset (src/coreclr + src/native): 700 -> 183 groups; the remaining groups are no raw/expanded pairs at one site. - redis: node and edge sets byte-identical. - New tests (RED without the change): extraction extract_c_misnested_raw_struct_replaced_by_flat_copy, extract_cpp_preprocessed_rescue_skips_raw_def_on_same_line, extract_cpp_rescue_keeps_expanded_class_when_raw_qn_is_taken (fixture delta-minimized from holder.h). - conditional_variants, extraction, c_lsp, pipeline, edge_types_probe, parse_coverage, lsp_resolution_probe: 1854 passed. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 485 +++++++++++++++++++++++++++++++++++++++- tests/test_extraction.c | 201 +++++++++++++++++ 2 files changed, 683 insertions(+), 3 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index bd0664ef6..a1a7b7c72 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2081,6 +2081,420 @@ static bool cbm_remap_preprocessed_def(CBMDefinition *def, const CBMPreprocessed return true; } +/* ── One definition, two parses: which copy the rescue keeps ────────────── + * + * The #961/#1989 rescue re-extracts definitions from the preprocessed parse to + * recover what the raw parse LOST. When an expanded def d and a raw def r sit + * at the same site (same label, name and start line, cbm_raw_def_same_site) + * but carry different QNs, the two parses scoped ONE definition differently, + * and adopting d minted a second node for it. One decision, in the rescue: + * + * - By default the RAW copy wins and d is not adopted. The expanded parse is + * no better judge of scope: it nests utilcode.h's classes inside + * CUnorderedArrayWithAllocator, puts rapidjson reader.h's StackStream and + * NumberStream specializations at file scope instead of inside + * GenericReader, and picks one of the `namespace SVR` / `namespace WKS` + * branches of coreclr gc's #ifdef. What the expanded pass gave d's QN + * (call and usage scopes, resolved targets, members only it saw) moves to + * r's QN (cbm_requalify_result). + * - EXCEPT when r's enclosing scope exists only because of a raw ERROR + * region the expanded parse resolved (cbm_misnested_raw_twin). A + * macro-generated body (`struct s { mixed_members };`, the kernel's + * lib/tests/randstruct_kunit.c) keeps the raw field list from closing, and + * every later struct of the file nests inside it; the expanded parse gets + * them flat. Then d REPLACES r, and whatever carried r's QN or sat under it + * (Fields, nested definitions, call/usage scopes, resolved member + * references) moves to d's QN (cbm_requalify_result). Every condition is + * required: + * - d lies in a raw ERROR region; + * - r's QN is d's scope plus at least one more segment: the raw parse + * nested the definition DEEPER under the same outer scope; + * - r's immediate raw parent p is a raw type definition whose span holds + * r, and the expanded parse has p too (same label, name and start + * line) but closes it BEFORE d starts; + * - a raw ERROR region lies between p's opening and d: the raw parse + * lost p's closing brace there; + * - the original text closes p's braces on exactly the line where the + * expanded parse closes p (comments, literals and directive lines + * skipped): the source itself ends p there, so no expanded error + * closed p early (a local expanded error, such as an attribute macro + * the expanded pass does not know, leaves that intact). + * A real nested type keeps its nesting: the expanded parse still holds it + * inside p, so r is not replaced, and r's own QN follows p's move. */ +/* Per expanded def of the rescue: CBM_PP_DEF_UNMAPPED, CBM_PP_DEF_NO_TWIN, or + * the index of the raw def it replaces. */ +enum { CBM_PP_DEF_UNMAPPED = -2, CBM_PP_DEF_NO_TWIN = -1 }; + +typedef struct { + const CBMDefinition *defs; /* raw defs [0, raw_count), expanded [raw_count, count) */ + int raw_count; + int count; + const int *pp_state; /* one per expanded def */ + const cbm_error_regions_t *raw_regs; /* original lines */ + const char *source; /* the original file */ + int source_len; +} cbm_twin_ctx_t; + +/* The losing copy's QN and the winner's. raw_wins: the expanded copy lost (the + * default); otherwise a mis-nested raw twin lost to its flat expanded copy. */ +typedef struct { + const char *from; + const char *to; + bool raw_wins; +} cbm_requalify_rule_t; + +/* Record that `from` now names `to`, allocating room for one rule per expanded + * def (`cap`) on first use; an allocation failure only skips the move. */ +static void cbm_requalify_push(cbm_requalify_rule_t **rules, int *n, int cap, const char *from, + const char *to, bool raw_wins) { + if (!from || !to || strcmp(from, to) == 0 || *n >= cap) { + return; + } + if (!*rules) { + *rules = (cbm_requalify_rule_t *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, + (size_t)cap * sizeof(cbm_requalify_rule_t)); + if (!*rules) { + return; + } + } + (*rules)[(*n)++] = (cbm_requalify_rule_t){from, to, raw_wins}; +} + +/* True when raw def `r` and the remapped expanded def `d` are one definition's + * site: same label and name, starting on the same original line. */ +static bool cbm_raw_def_same_site(const CBMDefinition *r, const CBMDefinition *d) { + return r->name && d->name && r->label && d->label && r->start_line == d->start_line && + strcmp(r->name, d->name) == 0 && strcmp(r->label, d->label) == 0; +} + +/* True when another raw def of a different label already holds raw def j's + * QN. A raw copy that lost its scope can collide with a misparsed raw def of + * that name (coreclr inc/holder.h: Class `AddressInitHolder` outside its + * `BaseWrapper` scope vs the raw parse's FORCEINLINE constructor read as a + * Function), and the graph keeps one node per QN -- "raw wins" would then + * keep neither the class nor its correctly scoped expanded copy. */ +static bool cbm_raw_qn_held_by_other_label(const CBMDefinition *defs, int count, int j) { + const CBMDefinition *r = &defs[j]; + if (!r->qualified_name || !r->label) { + return false; + } + for (int k = 0; k < count; k++) { + const CBMDefinition *o = &defs[k]; + if (k != j && o->qualified_name && o->label && + strcmp(o->qualified_name, r->qualified_name) == 0 && strcmp(o->label, r->label) != 0) { + return true; + } + } + return false; +} + +static bool cbm_regions_overlap(const cbm_error_regions_t *regs, uint32_t lo, uint32_t hi) { + for (int i = 0; i < regs->count; i++) { + if (regs->starts[i] <= hi && regs->ends[i] >= lo) { + return true; + } + } + return false; +} + +/* `qn` (length `len`) is `.`; returns the scope length, or 0. */ +static size_t cbm_qn_scope_len(const char *qn, size_t len, const char *name, size_t name_len) { + if (len <= name_len + 1 || qn[len - name_len - 1] != '.' || + strcmp(qn + len - name_len, name) != 0) { + return 0; + } + return len - name_len - 1; +} + +/* The raw type def whose QN is `r`'s scope (`scope_len` bytes of r's QN) and + * whose span holds r, or NULL. */ +static const CBMDefinition *cbm_misnested_raw_parent(const CBMDefinition *defs, int raw_count, + const CBMDefinition *r, size_t scope_len) { + for (int k = 0; k < raw_count; k++) { + const CBMDefinition *p = &defs[k]; + if (p != r && p->qualified_name && p->name && cbm_def_label_is_type(p->label) && + p->start_line <= r->start_line && p->end_line >= r->end_line && + strncmp(p->qualified_name, r->qualified_name, scope_len) == 0 && + p->qualified_name[scope_len] == '\0') { + return p; + } + } + return NULL; +} + +/* Index just past a preprocessor directive starting at `i` (its continuation + * lines included), and the lines it spans added to *line. */ +static int cbm_skip_directive(const char *src, int len, int i, uint32_t *line) { + while (i < len) { + if (src[i] != '\n') { + i++; + continue; + } + int b = i - 1; + if (b >= 0 && src[b] == '\r') { + b--; + } + (*line)++; + i++; + if (b < 0 || src[b] != '\\') { + break; + } + } + return i; +} + +/* Index just past a comment or a string/character literal starting at `i` + * (lines it spans added to *line), or `i` when none starts there. */ +static int cbm_skip_comment_or_literal(const char *src, int len, int i, uint32_t *line) { + if (src[i] == '/' && i + 1 < len && src[i + 1] == '/') { + while (i < len && src[i] != '\n') { + i++; + } + return i; + } + if (src[i] == '/' && i + 1 < len && src[i + 1] == '*') { + for (i += 2; i < len && !(src[i] == '*' && i + 1 < len && src[i + 1] == '/'); i++) { + *line += src[i] == '\n'; + } + return i + 2; + } + if (src[i] == '"' || src[i] == '\'') { + char quote = src[i]; + for (i++; i < len && src[i] != quote && src[i] != '\n'; i++) { + i += src[i] == '\\'; + } + return i + 1; + } + return i; +} + +/* True when the ORIGINAL text, read from the start of `start_line`, opens a + * brace and closes it again exactly on `end_line`: the source itself ends the + * definition there. Comments, literals and directive lines are skipped. */ +static bool cbm_source_braces_close_on(const char *src, int len, uint32_t start_line, + uint32_t end_line) { + uint32_t line = 1; + int i = 0; + for (; i < len && line < start_line; i++) { + line += src[i] == '\n'; + } + int depth = 0; + bool line_start = true; + while (i < len && line <= end_line) { + if (line_start) { + int k = i; + while (k < len && (src[k] == ' ' || src[k] == '\t')) { + k++; + } + if (k < len && src[k] == '#') { + i = cbm_skip_directive(src, len, k, &line); + continue; + } + line_start = false; + } + int next = cbm_skip_comment_or_literal(src, len, i, &line); + if (next != i) { + i = next; + continue; + } + if (src[i] == '\n') { + line++; + line_start = true; + } else if (src[i] == '{') { + depth++; + } else if (src[i] == '}' && --depth <= 0) { + return depth == 0 && line == end_line; + } + i++; + } + return false; +} + +/* True when raw parent `rp` encloses the expanded def at `di` only because of + * a raw ERROR region: the raw parse failed between rp's opening and that def, + * and the expanded parse has rp too but closes it before the def starts, on + * the line where the original text closes it. */ +static bool cbm_raw_parent_is_error_artifact(const cbm_twin_ctx_t *c, const CBMDefinition *rp, + int di) { + const CBMDefinition *d = &c->defs[di]; + if (!cbm_regions_overlap(c->raw_regs, rp->start_line, d->start_line)) { + return false; + } + for (int e = c->raw_count; e < c->count; e++) { + const CBMDefinition *ep = &c->defs[e]; + if (c->pp_state[e - c->raw_count] != CBM_PP_DEF_UNMAPPED && + ep->start_line == rp->start_line && ep->end_line < d->start_line && ep->name && + ep->label && strcmp(ep->name, rp->name) == 0 && strcmp(ep->label, rp->label) == 0 && + cbm_source_braces_close_on(c->source, c->source_len, ep->start_line, ep->end_line)) { + return true; + } + } + return false; +} + +/* Index of the raw def that the expanded def at `di` replaces under the rule + * above, or CBM_PP_DEF_NO_TWIN. */ +static int cbm_misnested_raw_twin(const cbm_twin_ctx_t *c, int di) { + const CBMDefinition *d = &c->defs[di]; + if (!d->name || !d->label || !d->qualified_name || d->qn_sig_off || + !cbm_regions_overlap(c->raw_regs, d->start_line, d->end_line)) { + return CBM_PP_DEF_NO_TWIN; + } + size_t name_len = strlen(d->name); + size_t d_scope = + cbm_qn_scope_len(d->qualified_name, strlen(d->qualified_name), d->name, name_len); + if (d_scope == 0) { + return CBM_PP_DEF_NO_TWIN; + } + for (int j = 0; j < c->raw_count; j++) { + const CBMDefinition *r = &c->defs[j]; + if (!cbm_raw_def_same_site(r, d) || !r->qualified_name || r->qn_sig_off) { + continue; + } + size_t r_scope = + cbm_qn_scope_len(r->qualified_name, strlen(r->qualified_name), d->name, name_len); + /* Deeper under d's own scope: ".". */ + if (r_scope <= d_scope + 1 || r->qualified_name[d_scope] != '.' || + strncmp(r->qualified_name, d->qualified_name, d_scope) != 0) { + continue; + } + const CBMDefinition *rp = cbm_misnested_raw_parent(c->defs, c->raw_count, r, r_scope); + if (rp && cbm_raw_parent_is_error_artifact(c, rp, di)) { + return j; + } + } + return CBM_PP_DEF_NO_TWIN; +} + +/* Remap every expanded def [raw_count, count) to original lines and find the + * raw twin each one replaces: pp_state[i - raw_count] becomes + * CBM_PP_DEF_UNMAPPED, CBM_PP_DEF_NO_TWIN or the twin's raw index. Returns the + * number of twins found. */ +static int cbm_prepare_preprocessed_defs(CBMFileResult *result, int raw_count, + const CBMPreprocessedSource *pp, + const cbm_error_regions_t *raw_regs, const char *source, + int source_len, int *pp_state) { + int count = result->defs.count; + CBMDefinition *defs = result->defs.items; + for (int i = raw_count; i < count; i++) { + pp_state[i - raw_count] = + cbm_remap_preprocessed_def(&defs[i], pp) ? CBM_PP_DEF_NO_TWIN : CBM_PP_DEF_UNMAPPED; + } + cbm_twin_ctx_t ctx = {defs, raw_count, count, pp_state, raw_regs, source, source_len}; + int twins = 0; + for (int i = raw_count; i < count; i++) { + if (pp_state[i - raw_count] == CBM_PP_DEF_UNMAPPED) { + continue; + } + int twin = cbm_misnested_raw_twin(&ctx, i); + if (twin >= 0) { + pp_state[i - raw_count] = twin; + twins++; + } + } + return twins; +} + +/* `qn` with the longest rule prefix ("" or ".") replaced + * by that rule's ""; `qn` itself when no rule applies. With + * `replacements_only`, raw_wins rules are not applied. */ +static const char *cbm_requalify_qn(CBMArena *a, const char *qn, const cbm_requalify_rule_t *rules, + int rule_count, bool replacements_only) { + if (!qn) { + return qn; + } + int best = -1; + size_t best_len = 0; + for (int i = 0; i < rule_count; i++) { + if (!rules[i].from || (replacements_only && rules[i].raw_wins)) { + continue; + } + size_t len = strlen(rules[i].from); + if (len > best_len && strncmp(qn, rules[i].from, len) == 0 && + (qn[len] == '\0' || qn[len] == '.')) { + best = i; + best_len = len; + } + } + if (best < 0) { + return qn; + } + const char *moved = cbm_arena_sprintf(a, "%s%s", rules[best].to, qn + best_len); + return moved ? moved : qn; +} + +/* Move everything the losing copies owned to the winners' QNs: definitions + * nested under them (Fields, nested types, expanded-only members) and every + * scope or target QN the result carries (call and usage scopes, resolved + * callers and targets), so no child and no edge is left on a QN with no node. + * A losing expanded QN that a kept definition still carries (a raw def at + * another site, a second expanded copy) leaves nothing dangling and does not + * move. A raw winner can itself sit under a mis-nested twin that moved: its + * final QN is resolved first. */ +static void cbm_requalify_result(CBMArena *a, CBMFileResult *r, cbm_requalify_rule_t *rules, + int n) { + for (int i = 0; i < n; i++) { + for (int k = 0; rules[i].raw_wins && k < r->defs.count; k++) { + const char *kept = r->defs.items[k].qualified_name; + if (kept && strcmp(kept, rules[i].from) == 0) { + rules[i].from = NULL; + break; + } + } + } + for (int i = 0; i < n; i++) { + if (rules[i].from && rules[i].raw_wins) { + rules[i].to = cbm_requalify_qn(a, rules[i].to, rules, n, true); + } + } + for (int i = 0; i < r->defs.count; i++) { + CBMDefinition *d = &r->defs.items[i]; + d->qualified_name = cbm_requalify_qn(a, d->qualified_name, rules, n, false); + d->parent_class = cbm_requalify_qn(a, d->parent_class, rules, n, false); + } + for (int i = 0; i < r->calls.count; i++) { + CBMCall *c = &r->calls.items[i]; + c->enclosing_func_qn = cbm_requalify_qn(a, c->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->usages.count; i++) { + CBMUsage *u = &r->usages.items[i]; + u->enclosing_func_qn = cbm_requalify_qn(a, u->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->throws.count; i++) { + CBMThrow *t = &r->throws.items[i]; + t->enclosing_func_qn = cbm_requalify_qn(a, t->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->rw.count; i++) { + CBMReadWrite *w = &r->rw.items[i]; + w->enclosing_func_qn = cbm_requalify_qn(a, w->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->type_refs.count; i++) { + CBMTypeRef *t = &r->type_refs.items[i]; + t->enclosing_func_qn = cbm_requalify_qn(a, t->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->env_accesses.count; i++) { + CBMEnvAccess *e = &r->env_accesses.items[i]; + e->enclosing_func_qn = cbm_requalify_qn(a, e->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->type_assigns.count; i++) { + CBMTypeAssign *t = &r->type_assigns.items[i]; + t->enclosing_func_qn = cbm_requalify_qn(a, t->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->string_refs.count; i++) { + CBMStringRef *s = &r->string_refs.items[i]; + s->enclosing_func_qn = cbm_requalify_qn(a, s->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->channels.count; i++) { + CBMChannel *c = &r->channels.items[i]; + c->enclosing_func_qn = cbm_requalify_qn(a, c->enclosing_func_qn, rules, n, false); + } + for (int i = 0; i < r->resolved_calls.count; i++) { + CBMResolvedCall *c = &r->resolved_calls.items[i]; + c->caller_qn = cbm_requalify_qn(a, c->caller_qn, rules, n, false); + c->callee_qn = cbm_requalify_qn(a, c->callee_qn, rules, n, false); + } +} + static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs, const char *src, int src_len) { if (regs->count <= 0) { @@ -3682,6 +4096,29 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C superseded = (char *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)defs_before); } + /* Remap every expanded def up front and find the + * mis-nested raw twins that flat expanded copies + * replace (cbm_misnested_raw_twin). A twin is + * dropped only once its replacement is adopted. */ + /* Counted through the context the extraction + * appended to (pp_ctx.result is `result`). */ + int pp_count = pp_ctx.result->defs.count - defs_before; + int *pp_state = pp_count > 0 + ? (int *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, + (size_t)pp_count * sizeof(int)) + : NULL; + int twin_count = pp_state ? cbm_prepare_preprocessed_defs( + result, defs_before, preprocessed, + &raw_regs, source, source_len, pp_state) + : 0; + char *misnested = + twin_count > 0 + ? (char *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)defs_before) + : NULL; + /* Losing copy -> winner, one per expanded def at + * most (cbm_requalify_push allocates on use). */ + cbm_requalify_rule_t *requalify = NULL; + int requalify_count = 0; /* #1989: spans of defs already adopted from the * expanded tree. A def nested inside one of these * (an inline method of a rescued class) was dropped @@ -3699,7 +4136,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C CBMDefinition *d = &result->defs.items[i]; bool adopt = false; int supersede_j = -1; - if (cbm_remap_preprocessed_def(d, preprocessed)) { + int pp_twin = + pp_state ? pp_state[i - defs_before] : CBM_PP_DEF_NO_TWIN; + bool mapped = pp_state + ? pp_twin != CBM_PP_DEF_UNMAPPED + : cbm_remap_preprocessed_def(d, preprocessed); + if (mapped) { for (int rj = 0; rj < raw_regs.count && !adopt; rj++) { if (d->start_line <= raw_regs.ends[rj] && d->end_line >= raw_regs.starts[rj]) { @@ -3800,12 +4242,42 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C } adopt = false; } + /* Same site, different QN: one definition the + * two parses scoped differently. The raw copy + * wins, unless it is the mis-nested twin this + * def replaces (cbm_misnested_raw_twin) or its + * QN is already another label's + * (cbm_raw_qn_held_by_other_label); what the + * expanded pass gave this copy's QN moves to + * the raw one. */ + if (adopt && j != pp_twin && + cbm_raw_def_same_site(&result->defs.items[j], d) && + !cbm_raw_qn_held_by_other_label(result->defs.items, + defs_before, j)) { + adopt = false; + cbm_requalify_push(&requalify, &requalify_count, pp_count, + d->qualified_name, + result->defs.items[j].qualified_name, + true); + } } if (adopt) { if (supersede_j >= 0 && supersede_j < defs_before && superseded) { superseded[supersede_j] = 1; } + /* The flat copy replaces its mis-nested raw + * twin; what sat under the twin's QN moves + * to this one below. Without room for that + * move the twin stays. */ + if (pp_twin >= 0 && misnested && !misnested[pp_twin]) { + int moves = requalify_count; + cbm_requalify_push( + &requalify, &requalify_count, pp_count, + result->defs.items[pp_twin].qualified_name, + d->qualified_name, false); + misnested[pp_twin] = (char)(requalify_count > moves); + } /* Phantom-artifact suppression (#1989): on the * raw tree `class MOD_API Foo : public Base * {...}` parses as a function_definition whose @@ -3843,10 +4315,11 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C result->defs.items[w++] = *d; } } - if (superseded) { + if (superseded || misnested) { int rw = 0; for (int j = 0; j < defs_before; j++) { - if (!superseded[j]) { + if (!(superseded && superseded[j]) && + !(misnested && misnested[j])) { result->defs.items[rw++] = result->defs.items[j]; } } @@ -3860,6 +4333,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_free(CBM_MEM_CLASS_EXTRACT, superseded); } result->defs.count = w; + if (requalify_count > 0) { + cbm_requalify_result(a, result, requalify, requalify_count); + } + cbm_free(CBM_MEM_CLASS_EXTRACT, misnested); + cbm_free(CBM_MEM_CLASS_EXTRACT, requalify); + cbm_free(CBM_MEM_CLASS_EXTRACT, pp_state); } } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 7382e147e..e39d40a8a 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7489,6 +7489,204 @@ TEST(extract_cpp_template_specialization_preprocessed_qn_single_spelling) { PASS(); } +/* Defs of `r` with this label, name and start line; `*qn` gets the last one's QN. */ +static int count_defs_at(const CBMFileResult *r, const char *label, const char *name, uint32_t line, + const char **qn) { + int n = 0; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0 && + d->start_line == line) { + *qn = d->qualified_name; + n++; + } + } + return n; +} + +/* A macro-generated struct body (the kernel's lib/tests/randstruct_kunit.c: + * `struct s { DO_MANY_MEMBERS(m) };`, `struct t { mixed_members };`) opens a + * raw ERROR region in which every later struct nests inside the broken one, + * while the preprocessed parse gets them flat. The #961 rescue adopted the + * flat copies next to the mis-nested raw ones: one definition, two QNs. The + * flat copy replaces the raw one, and what sat under the raw QN (Fields, a + * real nested struct, resolved member references) moves to the flat QN. The + * body's `__aligned(16)` is unknown without the kernel headers, as in the + * real file: a local error in the expanded parse that must not block it. */ +TEST(extract_c_misnested_raw_struct_replaced_by_flat_copy) { + CBMFileResult *r = extract("#define DO_MEMBERS(macro) macro(a) macro(b)\n" + "#define long_member(x) unsigned long x;\n" + "struct plain {\n" + "\tDO_MEMBERS(long_member)\n" + "};\n" + "\n" + "#define mixed_members \\\n" + "\tint a __aligned(16); \\\n" + "\tunion { \\\n" + "\t\tchar c; \\\n" + "\t\tlong g; \\\n" + "\t};\n" + "\n" + "struct mixed_untouched {\n" + "\tmixed_members\n" + "};\n" + "\n" + "struct mixed_shuffled {\n" + "\tmixed_members\n" + "} __randomize_layout;\n" + "\n" + "struct contains_untouched {\n" + "\tint before;\n" + "\tstruct nested_real {\n" + "\t\tint deep;\n" + "\t} real;\n" + "\tstruct mixed_shuffled inner;\n" + "\tint after;\n" + "};\n" + "\n" + "static int helper(int x)\n" + "{\n" + "\treturn x + 1;\n" + "}\n" + "\n" + "static int check_layout(struct contains_untouched *c)\n" + "{\n" + "\treturn helper(c->before) + c->real.deep;\n" + "}\n", + CBM_LANG_C, "t", "mixed.c"); + ASSERT_NOT_NULL(r); + static const struct { + const char *label; + const char *name; + uint32_t line; + const char *qn; + } want[] = { + {"Class", "mixed_shuffled", 18, "t.mixed.mixed_shuffled"}, + {"Class", "contains_untouched", 22, "t.mixed.contains_untouched"}, + {"Field", "before", 23, "t.mixed.contains_untouched.before"}, + {"Field", "after", 28, "t.mixed.contains_untouched.after"}, + /* A real nested struct keeps its real nesting. */ + {"Class", "nested_real", 24, "t.mixed.contains_untouched.nested_real"}, + {"Field", "deep", 25, "t.mixed.contains_untouched.nested_real.deep"}, + {"Function", "check_layout", 36, "t.mixed.check_layout"}, + }; + for (size_t i = 0; i < sizeof(want) / sizeof(want[0]); i++) { + const char *qn = NULL; + int n = count_defs_at(r, want[i].label, want[i].name, want[i].line, &qn); + if (n != 1 || !qn || strcmp(qn, want[i].qn) != 0) { + fprintf(stderr, " [misnested] %s %s L%u: %d def(s), last QN %s (want one, %s)\n", + want[i].label, want[i].name, want[i].line, n, qn ? qn : "(none)", want[i].qn); + } + ASSERT_EQ(n, 1); + ASSERT_STR_EQ(qn, want[i].qn); + } + /* Nothing is left on the mis-nested chain: no def, no call scope, no + * resolved caller or member target. */ + static const char *const chain = "mixed_untouched.mixed_shuffled"; + for (int i = 0; i < r->defs.count; i++) { + ASSERT_NULL(strstr(r->defs.items[i].qualified_name, chain)); + } + int helper_calls = 0; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + ASSERT_TRUE(!c->enclosing_func_qn || !strstr(c->enclosing_func_qn, chain)); + if (c->callee_name && strcmp(c->callee_name, "helper") == 0) { + ASSERT_STR_EQ(c->enclosing_func_qn, "t.mixed.check_layout"); + helper_calls++; + } + } + ASSERT_TRUE(helper_calls >= 1); + int before_refs = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *c = &r->resolved_calls.items[i]; + ASSERT_TRUE(!c->caller_qn || !strstr(c->caller_qn, chain)); + ASSERT_TRUE(!c->callee_qn || !strstr(c->callee_qn, chain)); + if (c->callee_qn && strcmp(c->callee_qn, "t.mixed.contains_untouched.before") == 0) { + before_refs++; + } + } + ASSERT_TRUE(before_refs >= 1); + cbm_free_result(r); + PASS(); +} + +/* The #961 rescue recovers defs the raw pass LOST. When the expanded tree scopes + * a def differently from the raw one, its QN differs although the raw pass + * already extracted the very same definition (same label, name, line). Here a + * namespace-opening macro defined in the file is an identifier to the raw parse + * and `namespace inner {` to the expanded one; rapidjson reader.h hits the same + * divergence on the `NumberStream<...>` specializations nested in GenericReader. + * Adopting the expanded copy mints a second node for one definition: the raw + * copy wins. */ +TEST(extract_cpp_preprocessed_rescue_skips_raw_def_on_same_line) { + CBMFileResult *r = extract("#define OPEN_NS namespace inner {\n" + "#define CLOSE_NS }\n" + "OPEN_NS\n" + "template struct box {};\n" + "template<> struct box {\n" + " static INLINE void first(int *p) { p[0] = 0; }\n" + "};\n" + "CLOSE_NS\n", + CBM_LANG_CPP, "t", "ns.h"); + ASSERT_NOT_NULL(r); + int first_count = 0; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->name && strcmp(d->name, "first") == 0) { + fprintf(stderr, " [rescue-site] def first: %s line %u\n", d->qualified_name, + d->start_line); + first_count++; + } + } + ASSERT_EQ(first_count, 1); + cbm_free_result(r); + PASS(); +} + +/* The raw copy wins a same-site disagreement only while its QN is its own: + * coreclr inc/holder.h's nested `class AddressInitHolder` loses its + * `BaseWrapper` scope on the raw parse, and the raw parse also reads the + * FORCEINLINE constructor as a file-scope Function of the same QN. Keeping the + * raw Class there left no Class node at all (one node per QN); the correctly + * scoped expanded copy must survive. */ +TEST(extract_cpp_rescue_keeps_expanded_class_when_raw_qn_is_taken) { + /* Minimized from holder.h (delta debugging against the rule with and + * without this check). */ + CBMFileResult *r = extract( + "class BaseWrapper : public BaseHolder\n" + "{\n" + " class AddressInitHolder\n" + " {\n" + " FORCEINLINE AddressInitHolder(BaseWrapper &holder)\n" + " {\n" + " unknown = static_cast(m_holder.m_value);\n" + " }\n" + "#if defined(FEATURE_COMINTEROP) && !defined(STRIKE)\n" + " FORCEINLINE operator IInspectable **()\n" + "#endif // FEATURE_COMINTEROP\n" + " {\n" + " }\n" + " }\n", + CBM_LANG_CPP, "t", "holder.h"); + ASSERT_NOT_NULL(r); + bool scoped_class = false; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->name && strcmp(d->name, "AddressInitHolder") == 0) { + fprintf(stderr, " [rescue-taken] %s %s line %u\n", d->label ? d->label : "-", + d->qualified_name, d->start_line); + if (d->label && strcmp(d->label, "Class") == 0 && d->qualified_name && + strstr(d->qualified_name, "BaseWrapper.AddressInitHolder") != NULL && + strstr(d->qualified_name, "AddressInitHolder.AddressInitHolder") == NULL) { + scoped_class = true; + } + } + } + ASSERT_TRUE(scoped_class); + cbm_free_result(r); + PASS(); +} + /* #1989: build-system export macros (UE "_API", CMake generate_export_header * "_EXPORT") are empty on the real compile line but opaque to tree-sitter, * so `class MOD_API Foo` misparses. The preprocessed second pass predefines the @@ -12425,6 +12623,9 @@ SUITE(extraction) { RUN_TEST(extract_c_ifdef_split_brace_after_include_remapped_issue949); RUN_TEST(extract_c_clean_file_no_recovery_duplicates_issue961); RUN_TEST(extract_cpp_template_specialization_preprocessed_qn_single_spelling); + RUN_TEST(extract_c_misnested_raw_struct_replaced_by_flat_copy); + RUN_TEST(extract_cpp_preprocessed_rescue_skips_raw_def_on_same_line); + RUN_TEST(extract_cpp_rescue_keeps_expanded_class_when_raw_qn_is_taken); RUN_TEST(extract_cpp_export_macro_class_recovery_issue1989); RUN_TEST(extract_cpp_export_macro_struct_recovery_issue1989); RUN_TEST(extract_cpp_export_macro_enum_recovery_issue1989);