diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 82a782d64..6af7dae40 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -43,6 +43,18 @@ #ifdef CBM_ENABLE_TEST_SEAMS _Atomic uint64_t cbm_py_submodule_probes = 0; _Atomic uint64_t cbm_py_submodule_probe_visits = 0; +_Atomic uint64_t cbm_py_module_class_binds = 0; +_Atomic uint64_t cbm_py_module_class_bind_visits = 0; +_Atomic uint64_t cbm_py_wildcard_invalidations = 0; +_Atomic uint64_t cbm_py_wildcard_invalidation_visits = 0; +static _Thread_local int py_module_scratch_allocations_left = -1; +static _Thread_local int py_module_prefix_allocations_left = -1; +void cbm_py_test_module_scratch_fail_after(int successful_allocations) { + py_module_scratch_allocations_left = successful_allocations; +} +void cbm_py_test_module_prefix_fail_after(int successful_allocations) { + py_module_prefix_allocations_left = successful_allocations; +} #endif #ifdef CBM_ENABLE_TEST_SEAMS @@ -4643,6 +4655,38 @@ static CBMScope *py_root_class_names(PyLSPContext *ctx, const PyKids *rk) { return names; } +/* The temporary query owner never supplies a retained binding/name/type. + * A disabled/failed optimization leaves the original module-aware scan usable. */ +static bool py_module_scratch_open(CBMArena *scratch) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_module_scratch_allocations_left == 0) + return false; + if (py_module_scratch_allocations_left > 0) + py_module_scratch_allocations_left--; +#endif + cbm_arena_init_lazy(scratch, 4096); + return true; +} + +static char *py_module_prefix(CBMArena *scratch, const char *module_qn, size_t length) { + /* The arena rounds up by seven bytes after this addition. */ + if (length > SIZE_MAX - 9) + return NULL; +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_module_prefix_allocations_left == 0) + return NULL; + if (py_module_prefix_allocations_left > 0) + py_module_prefix_allocations_left--; +#endif + char *prefix = cbm_arena_alloc(scratch, length + 2); + if (!prefix) + return NULL; + memcpy(prefix, module_qn, length); + prefix[length] = '.'; + prefix[length + 1] = '\0'; + return prefix; +} + /* A project registry can contain another file's class in the same logical * Python module. Keep those cross-file globals available, but do not prebind * classes declared by this source: their binding epoch belongs in the ordered @@ -4659,20 +4703,49 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) } size_t prefix_len = strlen(ctx->module_qn); bool package_scope = ctx->file_module_qn && strcmp(ctx->file_module_qn, ctx->module_qn) != 0; - CBMTypeShortIter all_types; - cbm_registry_all_types_chain(ctx->registry, &all_types); - for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) { - const CBMRegisteredType *type = &all_types.reg->types[i]; - const char *qn = type->qualified_name; - const char *name = type->short_name; - if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' || - cbm_scope_lookup_local(defined, name)) { - continue; + CBMArena scratch = {0}; + char *prefix = py_module_scratch_open(&scratch) + ? py_module_prefix(&scratch, ctx->module_qn, prefix_len) + : NULL; + uint64_t visits = 0; + for (const CBMTypeRegistry *link = ctx->registry; link; link = link->fallback) { + CBMQNPrefixIter it; + if (prefix) + cbm_registry_types_with_qn_prefix(link, prefix, package_scope, &scratch, &it); + int linear = 0; + for (;;) { + int i; + if (prefix) { + i = cbm_qn_prefix_iter_next(&it); + } else { + i = linear < link->type_count ? linear++ : -1; + if (i >= 0) + visits++; + } + if (i < 0) + break; + if (cbm_registry_type_shadowed(ctx->registry, link, i)) + continue; + const CBMRegisteredType *type = &link->types[i]; + const char *qn = type->qualified_name; + const char *name = type->short_name; + if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || + qn[prefix_len] != '.' || cbm_scope_lookup_local(defined, name)) + continue; + if (package_scope && strchr(qn + prefix_len + 1, '.')) + continue; /* submodule/nested classes are not package globals */ + py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); } - if (package_scope && strchr(qn + prefix_len + 1, '.')) - continue; /* submodule/nested classes are not package globals */ - py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); + if (prefix) + visits += it.visits; } + cbm_arena_destroy(&scratch); +#ifdef CBM_ENABLE_TEST_SEAMS + atomic_fetch_add_explicit(&cbm_py_module_class_binds, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&cbm_py_module_class_bind_visits, visits, memory_order_relaxed); +#else + (void)visits; +#endif } #ifdef CBM_ENABLE_TEST_SEAMS @@ -4746,19 +4819,47 @@ static void py_invalidate_registry_module_functions(PyLSPContext *ctx, if (!ctx || !registry || !ctx->module_qn) return; size_t prefix_len = strlen(ctx->module_qn); - for (int i = 0; i < registry->func_count; i++) { - const CBMRegisteredFunc *func = ®istry->funcs[i]; - const char *qn = func->qualified_name; - if (!qn || func->receiver_type || strncmp(qn, ctx->module_qn, prefix_len) != 0 || - qn[prefix_len] != '.') { - continue; + CBMArena scratch = {0}; + char *prefix = py_module_scratch_open(&scratch) + ? py_module_prefix(&scratch, ctx->module_qn, prefix_len) + : NULL; + uint64_t visits = 0; + for (const CBMTypeRegistry *link = registry; link; link = link->fallback) { + CBMQNPrefixIter it; + if (prefix) + cbm_registry_funcs_with_qn_prefix(link, prefix, true, &scratch, &it); + int linear = 0; + for (;;) { + int i; + if (prefix) { + i = cbm_qn_prefix_iter_next(&it); + } else { + i = linear < link->func_count ? linear++ : -1; + if (i >= 0) + visits++; + } + if (i < 0) + break; + const CBMRegisteredFunc *func = &link->funcs[i]; + const char *qn = func->qualified_name; + if (!qn || func->receiver_type || strncmp(qn, ctx->module_qn, prefix_len) != 0 || + qn[prefix_len] != '.') + continue; + const char *suffix = qn + prefix_len + 1; + if (!suffix[0] || strchr(suffix, '.')) + continue; + py_scope_bind(ctx, func->short_name ? func->short_name : suffix, cbm_type_unknown()); } - const char *suffix = qn + prefix_len + 1; - if (!suffix[0] || strchr(suffix, '.')) - continue; - py_scope_bind(ctx, func->short_name ? func->short_name : suffix, cbm_type_unknown()); + if (prefix) + visits += it.visits; } - py_invalidate_registry_module_functions(ctx, registry->fallback); + cbm_arena_destroy(&scratch); +#ifdef CBM_ENABLE_TEST_SEAMS + atomic_fetch_add_explicit(&cbm_py_wildcard_invalidations, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&cbm_py_wildcard_invalidation_visits, visits, memory_order_relaxed); +#else + (void)visits; +#endif } /* `from module import *` may overwrite any module global through `__all__`. @@ -5749,6 +5850,7 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i } py_mark_ambiguous_callable_bindings(reg, arena); + cbm_registry_build_type_qn_order(reg, arena); cbm_registry_finalize(reg); reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */ return reg; diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 016beccdf..fa5e03e06 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -250,6 +250,15 @@ bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ * Only missing exact symbol/type attributes reach this submodule predicate. */ extern _Atomic uint64_t cbm_py_submodule_probes; extern _Atomic uint64_t cbm_py_submodule_probe_visits; +/* Complete module operations, including optimization fallbacks. Visits count + * inspected QNs/search comparisons, not sorting or string-length work. */ +extern _Atomic uint64_t cbm_py_module_class_binds; +extern _Atomic uint64_t cbm_py_module_class_bind_visits; +extern _Atomic uint64_t cbm_py_wildcard_invalidations; +extern _Atomic uint64_t cbm_py_wildcard_invalidation_visits; +/* -1 disables; zero fails until reset; positive counts down attempts. */ +void cbm_py_test_module_scratch_fail_after(int successful_allocations); +void cbm_py_test_module_prefix_fail_after(int successful_allocations); #endif #endif // CBM_LSP_PY_LSP_H diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 76e55f1e0..e943c2522 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -1008,7 +1008,8 @@ void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena) { reg->func_qn_sorted = NULL; reg->func_qn_sorted_count = 0; reg->func_qn_sorted_upto = 0; - if (reg->func_count <= 0 || (size_t)reg->func_count > SIZE_MAX / sizeof(CBMRegistryQNEntry)) + if (reg->func_count <= 0 || + (size_t)reg->func_count > (SIZE_MAX - 7) / sizeof(CBMRegistryQNEntry)) return; #ifdef CBM_ENABLE_TEST_SEAMS if (registry_qn_order_allocations_left == 0) @@ -1079,6 +1080,205 @@ bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *pre return false; } +static int registry_type_qn_entry_cmp(const void *left, const void *right) { + const CBMRegistryTypeQNEntry *a = left; + const CBMRegistryTypeQNEntry *b = right; + int cmp = strcmp(a->qualified_name, b->qualified_name); + return cmp ? cmp : (a->type_index > b->type_index) - (a->type_index < b->type_index); +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int registry_type_qn_order_allocations_left = -1; +static _Thread_local int registry_qn_prefix_block_allocations_left = -1; +void cbm_registry_test_type_qn_order_fail_after(int successful_allocations) { + registry_type_qn_order_allocations_left = successful_allocations; +} +void cbm_registry_test_qn_prefix_block_fail_after(int successful_allocations) { + registry_qn_prefix_block_allocations_left = successful_allocations; +} +#endif + +void cbm_registry_build_type_qn_order(CBMTypeRegistry *reg, CBMArena *arena) { + if (!reg || !arena || reg->read_only) + return; + reg->type_qn_sorted = NULL; + reg->type_qn_sorted_count = 0; + reg->type_qn_sorted_upto = 0; + if (!reg->types || reg->type_count <= 0 || + (size_t)reg->type_count > (SIZE_MAX - 7) / sizeof(CBMRegistryTypeQNEntry)) + return; +#ifdef CBM_ENABLE_TEST_SEAMS + if (registry_type_qn_order_allocations_left == 0) + return; + if (registry_type_qn_order_allocations_left > 0) + registry_type_qn_order_allocations_left--; +#endif + CBMRegistryTypeQNEntry *sorted = + cbm_arena_alloc(arena, (size_t)reg->type_count * sizeof(*sorted)); + if (!sorted) + return; + int count = 0; + for (int i = 0; i < reg->type_count; i++) { + if (reg->types[i].qualified_name) { + sorted[count].qualified_name = reg->types[i].qualified_name; + sorted[count].type_index = i; + count++; + } + } + qsort(sorted, (size_t)count, sizeof(*sorted), registry_type_qn_entry_cmp); + reg->type_qn_sorted = sorted; + reg->type_qn_sorted_count = count; + reg->type_qn_sorted_upto = reg->type_count; +} + +bool cbm_registry_type_shadowed(const CBMTypeRegistry *head, const CBMTypeRegistry *reg, + int index) { + return reg && reg->types && index >= 0 && index < reg->type_count && + type_shadowed(head, reg, index); +} + +static const char *registry_prefix_sorted_qn(const CBMQNPrefixIter *it, int index) { + return it->types ? it->reg->type_qn_sorted[index].qualified_name + : it->reg->func_qn_sorted[index].qualified_name; +} + +static int registry_prefix_sorted_index(const CBMQNPrefixIter *it, int index) { + return it->types ? it->reg->type_qn_sorted[index].type_index + : it->reg->func_qn_sorted[index].func_index; +} + +static int registry_prefix_index_cmp(const void *left, const void *right) { + int a = *(const int *)left, b = *(const int *)right; + return (a > b) - (a < b); +} + +/* Count/fill the exact range. For a direct query, jump over dotted subtrees + * with a binary upper bound on the prefix INCLUDING its separating dot. The + * undotted siblings A-b and AB must remain visible around A.child. */ +static int registry_prefix_collect(CBMQNPrefixIter *it, int start, int sorted_count, int covered, + int *indices, int capacity) { + int count = 0; + for (int pos = start; pos < sorted_count;) { + const char *qn = registry_prefix_sorted_qn(it, pos); + it->visits++; + if (!qn) + return -1; + if (strncmp(qn, it->prefix, it->prefix_len) != 0) + break; + const char *dot = it->direct_only ? strchr(qn + it->prefix_len, '.') : NULL; + if (dot) { + size_t subtree_len = (size_t)(dot - qn) + 1; + int lo = pos + 1, hi = sorted_count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + const char *candidate = registry_prefix_sorted_qn(it, mid); + it->visits++; + if (!candidate) + return -1; + if (strncmp(candidate, qn, subtree_len) <= 0) + lo = mid + 1; + else + hi = mid; + } + pos = lo; + continue; + } + int index = registry_prefix_sorted_index(it, pos++); + if (index < 0 || index >= covered || (indices && count >= capacity)) + return -1; + if (indices) + indices[count] = index; + count++; + } + return count; +} + +static bool registry_prefix_open(const CBMTypeRegistry *reg, const char *prefix, bool direct_only, + bool types, CBMArena *scratch, CBMQNPrefixIter *out) { + if (!out) + return false; + memset(out, 0, sizeof(*out)); + out->reg = reg; + out->prefix = prefix; + out->types = types; + out->direct_only = direct_only; + if (!reg || !prefix || (types ? !reg->types : !reg->funcs)) + return true; + out->prefix_len = strlen(prefix); + int total = types ? reg->type_count : reg->func_count; + out->tail_end = total > 0 ? total : 0; + int covered = types ? reg->type_qn_sorted_upto : reg->func_qn_sorted_upto; + int sorted_count = types ? reg->type_qn_sorted_count : reg->func_qn_sorted_count; + bool has_order = types ? reg->type_qn_sorted != NULL : reg->func_qn_sorted != NULL; + if (!has_order || covered < 0 || covered > total || sorted_count < 0 || sorted_count > covered) + return true; /* initialized full-link fallback */ + + int lo = 0, hi = sorted_count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + const char *qn = registry_prefix_sorted_qn(out, mid); + out->visits++; + if (!qn) + return true; + if (strcmp(qn, prefix) < 0) + lo = mid + 1; + else + hi = mid; + } + int count = registry_prefix_collect(out, lo, sorted_count, covered, NULL, 0); + if (count < 0) + return true; + if (count == 0) { + out->tail_i = covered; + return true; + } + if (!scratch || (size_t)count > (SIZE_MAX - 7) / sizeof(int)) + return true; +#ifdef CBM_ENABLE_TEST_SEAMS + if (registry_qn_prefix_block_allocations_left == 0) + return true; + if (registry_qn_prefix_block_allocations_left > 0) + registry_qn_prefix_block_allocations_left--; +#endif + int *indices = cbm_arena_alloc(scratch, (size_t)count * sizeof(*indices)); + if (!indices || + registry_prefix_collect(out, lo, sorted_count, covered, indices, count) != count) + return true; + qsort(indices, (size_t)count, sizeof(*indices), registry_prefix_index_cmp); + /* Publish only a complete block; every earlier return scans from zero. */ + out->indices = indices; + out->count = count; + out->tail_i = covered; + return true; +} + +bool cbm_registry_types_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out) { + return registry_prefix_open(reg, prefix, direct_only, true, scratch, out); +} + +bool cbm_registry_funcs_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out) { + return registry_prefix_open(reg, prefix, direct_only, false, scratch, out); +} + +int cbm_qn_prefix_iter_next(CBMQNPrefixIter *it) { + if (!it || !it->reg || !it->prefix) + return -1; + if (it->pos < it->count) + return it->indices[it->pos++]; + while (it->tail_i < it->tail_end) { + int index = it->tail_i++; + const char *qn = + it->types ? it->reg->types[index].qualified_name : it->reg->funcs[index].qualified_name; + it->visits++; + if (qn && strncmp(qn, it->prefix, it->prefix_len) == 0 && + (!it->direct_only || !strchr(qn + it->prefix_len, '.'))) + return index; + } + return -1; +} + // Count parameters in a FUNC signature. static int count_func_params(const CBMRegisteredFunc *f) { if (!f || !f->signature || f->signature->kind != CBM_TYPE_FUNC) diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index 0d1ef564b..836f65f87 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -89,6 +89,12 @@ typedef struct { int func_index; } CBMRegistryQNEntry; +/* Type identities use their own representation; func_index remains unchanged. */ +typedef struct { + const char *qualified_name; + int type_index; +} CBMRegistryTypeQNEntry; + // Cross-file type/function registry. typedef struct CBMTypeRegistry { CBMRegisteredFunc *funcs; @@ -172,6 +178,9 @@ typedef struct CBMTypeRegistry { CBMRegistryQNEntry *func_qn_sorted; int func_qn_sorted_count; int func_qn_sorted_upto; + CBMRegistryTypeQNEntry *type_qn_sorted; + int type_qn_sorted_count; + int type_qn_sorted_upto; } CBMTypeRegistry; // Initialize a registry. @@ -239,6 +248,46 @@ bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *pre void cbm_registry_test_qn_order_fail_after(int successful_allocations); #endif +/* Optional type order, with the same ownership/immutability/rebuild contract as + * the function order. Build shared orders before sealing; queries never build. */ +void cbm_registry_build_type_qn_order(CBMTypeRegistry *reg, CBMArena *arena); + +/* Single-link literal-prefix query. Results retain original array order, + * including duplicate QNs, followed by the captured unindexed tail. Registry + * identities and the borrowed prefix must stay alive and unchanged while open; + * no concurrent registry mutation. No movable element pointers are retained. + * direct_only excludes a dot anywhere after the prefix (an empty suffix matches). + * With a non-NULL out, open always succeeds: missing inputs are empty; missing + * scratch, an invalid snapshot or optimization OOM selects the full linear scan. + * NULL out returns false; next(NULL) returns -1. Caller owns/destroys scratch. + * visits includes compared/inspected QNs in search, count, fill and tail passes, + * but excludes string-length work and sorting the original-index block. */ +typedef struct { + const CBMTypeRegistry *reg; + const char *prefix; + size_t prefix_len; + bool types; + bool direct_only; + int *indices; + int count; + int pos; + int tail_i; + int tail_end; + uint64_t visits; +} CBMQNPrefixIter; + +bool cbm_registry_types_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out); +bool cbm_registry_funcs_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out); +int cbm_qn_prefix_iter_next(CBMQNPrefixIter *it); +/* Preserve the existing chain iterator's original-head-only type shadow rule. */ +bool cbm_registry_type_shadowed(const CBMTypeRegistry *head, const CBMTypeRegistry *reg, int index); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_registry_test_type_qn_order_fail_after(int successful_allocations); +void cbm_registry_test_qn_prefix_block_fail_after(int successful_allocations); +#endif + // Resolve type alias chain: follow alias_of until concrete type found (max 16 levels). const CBMRegisteredType *cbm_registry_resolve_alias(const CBMTypeRegistry *reg, const char *type_qn); diff --git a/tests/test_complexity.c b/tests/test_complexity.c index 758bc7c88..f52a11ab9 100644 --- a/tests/test_complexity.c +++ b/tests/test_complexity.c @@ -462,6 +462,15 @@ typedef struct { int pyext_files; int pyext_defs; int pyext_controls; + uint64_t py_class_binds; + uint64_t py_class_visits; + uint64_t py_wildcards; + uint64_t py_wildcard_visits; + int pyrange_files; + int pyrange_defs; + int pyrange_classes; + int pyrange_peer_classes; + int pyrange_controls; int errors; double wall_s; } CxMetrics; @@ -532,13 +541,170 @@ static bool cx_pyext_population(cbm_store_t *store, const char *project, CxMetri return ok; } +/* Each package has many registered direct members; other packages are decoys. + * Wildcards precede definitions so the fixture retains explicit LSP positives. + * Exact invalidation-order negatives live in test_py_lsp.c. */ +enum { CX_PYRANGE_FILES_PER_K = 20, CX_PYRANGE_DEFS = 40, CX_PYRANGE_WILDCARDS = 4 }; + +static int cx_build_pyrange(const char *root, int k) { + for (int m = 0; m < k; m++) { + for (int j = 0; j < CX_PYRANGE_FILES_PER_K; j++) { + char dir[1024], path[1200]; + int dn = snprintf(dir, sizeof(dir), "%s/pyrange/mod%d/p%d", root, m, j); + if (dn < 0 || (size_t)dn >= sizeof(dir) || th_mkdir_p(dir) != 0) + return -1; + int pn = snprintf(path, sizeof(path), "%s/__init__.py", dir); + if (pn < 0 || (size_t)pn >= sizeof(path)) + return -1; + FILE *f = fopen(path, "w"); + if (!f) + return -1; + bool written = true; + for (int w = 0; written && w < CX_PYRANGE_WILDCARDS; w++) + written = fprintf(f, "from range_vendor_%d import *\n", w) >= 0; + int peer = (j + 1) % CX_PYRANGE_FILES_PER_K; + if (written) + written = fprintf(f, "from pyrange.mod%d.p%d import Peer_%d_%d\n\n", m, peer, m, + peer) >= 0; + for (int d = 0; written && d < CX_PYRANGE_DEFS; d++) { + written = fprintf(f, + "def cx_pyrange_definition_%d(x):\n return x + %d\n\n" + "class RangeClass_%d:\n def value(self):\n" + " return %d\n\n", + d, d, d, d) >= 0; + } + if (written) + written = fprintf(f, + "class Peer_%d_%d:\n def remote_%d_%d(self):\n" + " return %d\n\n" + "def cx_pyrange_use():\n obj = RangeClass_0()\n" + " peer = Peer_%d_%d()\n" + " return (cx_pyrange_definition_0(1) + obj.value()\n" + " + peer.remote_%d_%d())\n", + m, j, m, j, j, m, peer, m, peer) >= 0; + int closed = fclose(f); + if (!written || closed != 0) + return -1; + } + } + return 0; +} + +static bool cx_pyrange_exact_control(cbm_store_t *store, const char *project, + const cbm_node_t *caller, const char *qn, + const char *target_file, const char *label) { + if (!caller->qualified_name || !caller->file_path || !caller->label || + strcmp(caller->label, "Function") != 0) + return false; + cbm_node_t target = {0}; + cbm_edge_t *edges = NULL; + int count = 0, exact = 0; + bool ok = cbm_store_find_node_by_qn(store, project, qn, &target) == CBM_STORE_OK && + target.qualified_name && strcmp(target.qualified_name, qn) == 0 && target.label && + strcmp(target.label, label) == 0 && target.file_path && + strcmp(target.file_path, target_file) == 0 && + cbm_store_find_edges_by_source_type(store, caller->id, "CALLS", &edges, &count) == + CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + if (edges[i].target_id != target.id) + continue; + yyjson_doc *doc = + edges[i].properties_json + ? yyjson_read(edges[i].properties_json, strlen(edges[i].properties_json), 0) + : NULL; + const char *source = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "source")) : NULL; + if (source && strcmp(source, "lsp") == 0) + exact++; + yyjson_doc_free(doc); + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&target); + return ok && exact == 1; +} + +/* A remote method has no declaration in the importer/raw file registry. Its + * source=lsp edge proves that cross-file registry evidence reached resolution. */ +static bool cx_pyrange_controls(cbm_store_t *store, const char *project, const cbm_node_t *caller) { + if (!caller->qualified_name || !caller->file_path) + return false; + const char *dot = strrchr(caller->qualified_name, '.'); + if (!dot || dot - caller->qualified_name > 1000) + return false; + char module[1200], qn[1400]; + int n = snprintf(module, sizeof(module), "%.*s", (int)(dot - caller->qualified_name), + caller->qualified_name); + if (n < 0 || (size_t)n >= sizeof(module)) + return false; + n = snprintf(qn, sizeof(qn), "%s.cx_pyrange_definition_0", module); + bool function = + n >= 0 && (size_t)n < sizeof(qn) && + cx_pyrange_exact_control(store, project, caller, qn, caller->file_path, "Function"); + n = snprintf(qn, sizeof(qn), "%s.RangeClass_0.value", module); + bool method = n >= 0 && (size_t)n < sizeof(qn) && + cx_pyrange_exact_control(store, project, caller, qn, caller->file_path, "Method"); + int m = -1, j = -1; + if (sscanf(caller->file_path, "pyrange/mod%d/p%d/__init__.py", &m, &j) != 2 || m < 0 || j < 0 || + j >= CX_PYRANGE_FILES_PER_K) + return false; + char path[1200], slot_name[64]; + n = snprintf(path, sizeof(path), "pyrange/mod%d/p%d/__init__.py", m, j); + if (n < 0 || (size_t)n >= sizeof(path) || strcmp(path, caller->file_path) != 0) + return false; + n = snprintf(slot_name, sizeof(slot_name), "p%d", j); + char *slot = strrchr(module, '.'); + if (n < 0 || (size_t)n >= sizeof(slot_name) || !slot || strcmp(slot + 1, slot_name) != 0) + return false; + *slot = '\0'; + int peer = (j + 1) % CX_PYRANGE_FILES_PER_K; + n = snprintf(qn, sizeof(qn), "%s.p%d.Peer_%d_%d.remote_%d_%d", module, peer, m, peer, m, peer); + int pn = snprintf(path, sizeof(path), "pyrange/mod%d/p%d/__init__.py", m, peer); + bool remote = n >= 0 && (size_t)n < sizeof(qn) && pn >= 0 && (size_t)pn < sizeof(path) && + strcmp(path, caller->file_path) != 0 && + cx_pyrange_exact_control(store, project, caller, qn, path, "Method"); + return function && method && remote; +} + +static bool cx_pyrange_population(cbm_store_t *store, const char *project, CxMetrics *out) { + static const char *const labels[] = {"File", "Function", "Class"}; + bool ok = true; + for (int l = 0; l < 3; l++) { + cbm_node_t *nodes = NULL; + int count = 0; + bool queried = cbm_store_find_nodes_by_label(store, project, labels[l], &nodes, &count) == + CBM_STORE_OK; + ok = queried && ok; + for (int i = 0; queried && i < count; i++) { + const cbm_node_t *node = &nodes[i]; + if (!node->file_path || strncmp(node->file_path, "pyrange/", 8) != 0) + continue; + if (l == 0) + out->pyrange_files++; + if (l == 1 && node->name) { + static const char def_prefix[] = "cx_pyrange_definition_"; + if (strncmp(node->name, def_prefix, sizeof(def_prefix) - 1) == 0) + out->pyrange_defs++; + if (strcmp(node->name, "cx_pyrange_use") == 0 && + cx_pyrange_controls(store, project, node)) + out->pyrange_controls++; + } + if (l == 2 && node->name && strncmp(node->name, "RangeClass_", 11) == 0) + out->pyrange_classes++; + if (l == 2 && node->name && strncmp(node->name, "Peer_", 5) == 0) + out->pyrange_peer_classes++; + } + cbm_store_free_nodes(nodes, count); + } + return ok; +} + static double cx_now_s(void) { struct timespec ts; cbm_profile_now(&ts); return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; } -static int cx_run(const char *root, const char *db_path, CxMetrics *out, bool pyext) { +static int cx_run(const char *root, const char *db_path, CxMetrics *out, bool pyext, bool pyrange) { memset(out, 0, sizeof(*out)); uint64_t d0; @@ -558,6 +724,11 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out, bool py uint64_t dh0 = atomic_load_explicit(&g_dk_hub_edges, memory_order_relaxed); uint64_t pp0 = atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); uint64_t pv0 = atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); + uint64_t pc0 = atomic_load_explicit(&cbm_py_module_class_binds, memory_order_relaxed); + uint64_t pcv0 = atomic_load_explicit(&cbm_py_module_class_bind_visits, memory_order_relaxed); + uint64_t pw0 = atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed); + uint64_t pwv0 = + atomic_load_explicit(&cbm_py_wildcard_invalidation_visits, memory_order_relaxed); atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed); cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE); @@ -620,7 +791,16 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out, bool py out->infra_nodes[i] = cbm_store_count_nodes_scoped(s, project, CX_INFRA_DIRS[i]); out->infra_edges[i] = cbm_store_count_edges_scoped(s, project, CX_INFRA_DIRS[i]); } - bool extra_ok = !pyext || cx_pyext_population(s, project, out); + out->py_class_binds = + atomic_load_explicit(&cbm_py_module_class_binds, memory_order_relaxed) - pc0; + out->py_class_visits = + atomic_load_explicit(&cbm_py_module_class_bind_visits, memory_order_relaxed) - pcv0; + out->py_wildcards = + atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed) - pw0; + out->py_wildcard_visits = + atomic_load_explicit(&cbm_py_wildcard_invalidation_visits, memory_order_relaxed) - pwv0; + bool extra_ok = (!pyext || cx_pyext_population(s, project, out)) && + (!pyrange || cx_pyrange_population(s, project, out)); cbm_store_close(s); return extra_ok ? 0 : -1; } @@ -631,10 +811,12 @@ static double cx_ratio(double num, double den) { typedef int (*CxCorpusBuilder)(const char *root, int k); -static void cx_remove_corpus(const char *root) { - if (th_rmtree(root) != 0) { +static int cx_remove_corpus(const char *root) { + int rc = th_rmtree(root); + if (rc != 0) { fprintf(stderr, " [complexity] WARN: could not remove temp corpus %s\n", root); } + return rc; } /* Build one corpus shape at k and 2k in two fresh temp dirs, run the pipeline @@ -647,27 +829,30 @@ static int cx_measure(CxCorpusBuilder build, const char *prefix_a, const char *p const char *tmp = cbm_tmpdir(); char root_a[512]; char root_b[512]; - snprintf(root_a, sizeof(root_a), "%s/%s_XXXXXX", tmp, prefix_a); - snprintf(root_b, sizeof(root_b), "%s/%s_XXXXXX", tmp, prefix_b); + int na = snprintf(root_a, sizeof(root_a), "%s/%s_XXXXXX", tmp, prefix_a); + int nb = snprintf(root_b, sizeof(root_b), "%s/%s_XXXXXX", tmp, prefix_b); + if (na < 0 || (size_t)na >= sizeof(root_a) || nb < 0 || (size_t)nb >= sizeof(root_b)) + return -1; bool made_a = cbm_mkdtemp(root_a) != NULL; bool made_b = made_a && cbm_mkdtemp(root_b) != NULL; int rc = -1; if (made_b && build(root_a, CX_K_BASE) == 0 && build(root_b, CX_K_BASE * 2) == 0) { char db_a[600]; char db_b[600]; - snprintf(db_a, sizeof(db_a), "%s/cx.db", root_a); - snprintf(db_b, sizeof(db_b), "%s/cx.db", root_b); + int da = snprintf(db_a, sizeof(db_a), "%s/cx.db", root_a); + int db = snprintf(db_b, sizeof(db_b), "%s/cx.db", root_b); + bool paths = da >= 0 && (size_t)da < sizeof(db_a) && db >= 0 && (size_t)db < sizeof(db_b); bool pyext = build == cx_build_pyext; - if (cx_run(root_a, db_a, out_a, pyext) == 0 && cx_run(root_b, db_b, out_b, pyext) == 0) { + bool pyrange = build == cx_build_pyrange; + if (paths && cx_run(root_a, db_a, out_a, pyext, pyrange) == 0 && + cx_run(root_b, db_b, out_b, pyext, pyrange) == 0) { rc = 0; } } - if (made_b) { - cx_remove_corpus(root_b); - } - if (made_a) { - cx_remove_corpus(root_a); - } + if (made_b && cx_remove_corpus(root_b) != 0 && build == cx_build_pyrange) + rc = -1; + if (made_a && cx_remove_corpus(root_a) != 0 && build == cx_build_pyrange) + rc = -1; return rc; } @@ -1141,7 +1326,68 @@ TEST(complexity_rescript_error_recovery_lexing_is_linear) { PASS(); } +TEST(complexity_py_module_ranges_preserve_calls_with_bounded_work) { + static const char *const keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {NULL, NULL}; + bool ready = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + if (old && !saved[i]) + ready = false; + } + CxMetrics a = {0}, b = {0}; + int rc = -1, restored = 0; + if (ready) { + int configured = cbm_setenv(keys[0], "2", 1); + configured |= cbm_unsetenv(keys[1]); + if (configured == 0) + rc = cx_measure(cx_build_pyrange, "cbm_cxrange_a", "cbm_cxrange_b", &a, &b); + for (int i = 0; i < 2; i++) + restored |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + } + for (int i = 0; i < 2; i++) + free(saved[i]); + /* Both owned corpora are removed and environment restored before assertions. */ + ASSERT_TRUE(ready); + ASSERT_EQ(restored, 0); + ASSERT_EQ(rc, 0); + const int files = CX_K_BASE * CX_PYRANGE_FILES_PER_K; + ASSERT_GT(files, 50); + ASSERT_EQ(a.errors, 0); + ASSERT_EQ(b.errors, 0); + ASSERT_EQ(a.pyrange_files, files); + ASSERT_EQ(b.pyrange_files, files * 2); + ASSERT_EQ(a.pyrange_defs, files * CX_PYRANGE_DEFS); + ASSERT_EQ(b.pyrange_defs, files * 2 * CX_PYRANGE_DEFS); + ASSERT_EQ(a.pyrange_classes, files * CX_PYRANGE_DEFS); + ASSERT_EQ(b.pyrange_classes, files * 2 * CX_PYRANGE_DEFS); + ASSERT_EQ(a.pyrange_peer_classes, files); + ASSERT_EQ(b.pyrange_peer_classes, files * 2); + ASSERT_EQ(a.pyrange_controls, files); + ASSERT_EQ(b.pyrange_controls, files * 2); + ASSERT_GTE(a.py_class_binds, (uint64_t)files); + ASSERT_GTE(b.py_class_binds, (uint64_t)(files * 2)); + ASSERT_GTE(a.py_wildcards, (uint64_t)(files * CX_PYRANGE_WILDCARDS)); + ASSERT_GTE(b.py_wildcards, (uint64_t)(files * 2 * CX_PYRANGE_WILDCARDS)); + ASSERT_GT(a.py_class_visits, (uint64_t)0); + ASSERT_GT(b.py_class_visits, (uint64_t)0); + ASSERT_GT(a.py_wildcard_visits, (uint64_t)0); + ASSERT_GT(b.py_wildcard_visits, (uint64_t)0); + double classes = cx_ratio((double)b.py_class_binds, (double)a.py_class_binds); + double wildcards = cx_ratio((double)b.py_wildcards, (double)a.py_wildcards); + ASSERT_TRUE(classes >= CX_RATIO_LO && classes <= CX_RATIO_HI); + ASSERT_TRUE(wildcards >= CX_RATIO_LO && wildcards <= CX_RATIO_HI); + ASSERT_TRUE(cx_ratio((double)b.py_class_visits, (double)a.py_class_visits) <= CX_RATIO_HI); + ASSERT_TRUE(cx_ratio((double)b.py_wildcard_visits, (double)a.py_wildcard_visits) <= + CX_RATIO_HI); + /* These counters exclude sorting/string-length work. The direct registry + * tests separately pin absolute bounds and exact ordered answers. */ + PASS(); +} + SUITE(complexity) { + RUN_TEST(complexity_py_module_ranges_preserve_calls_with_bounded_work); RUN_TEST(complexity_rescript_error_recovery_lexing_is_linear); RUN_TEST(complexity_replicated_modules_scale_linearly); RUN_TEST(complexity_perfile_registry_work_is_linear); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index cb2542d45..7c177da92 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -4299,7 +4299,467 @@ TEST(pylsp_prefix_raw_per_file_and_shared_keep_exact_calls) { PASS(); } +/* Module-range queries must return the old array sequence, including duplicates. */ +static bool pylsp_range_add_type(CBMTypeRegistry *reg, const char *qn, const char *name) { + int before = reg->type_count; + cbm_registry_add_type(reg, (CBMRegisteredType){.qualified_name = qn, .short_name = name}); + return reg->type_count == before + 1; +} + +static bool pylsp_range_member(const char *qn, const char *prefix, bool direct) { + if (!qn || !prefix) + return false; + size_t n = strlen(prefix); + return strncmp(qn, prefix, n) == 0 && (!direct || !strchr(qn + n, '.')); +} + +/* No fixed result buffer and no call to the implementation's member predicate. */ +static bool pylsp_range_oracle(const CBMTypeRegistry *reg, bool types, const char *prefix, + bool direct, CBMArena *scratch, uint64_t *visits) { + CBMQNPrefixIter it; + bool opened = types ? cbm_registry_types_with_qn_prefix(reg, prefix, direct, scratch, &it) + : cbm_registry_funcs_with_qn_prefix(reg, prefix, direct, scratch, &it); + if (!opened) + return false; + int total = reg ? (types ? reg->type_count : reg->func_count) : 0; + bool ok = true; + for (int i = 0; i < total; i++) { + const char *qn = types ? reg->types[i].qualified_name : reg->funcs[i].qualified_name; + if (pylsp_range_member(qn, prefix, direct)) + ok = cbm_qn_prefix_iter_next(&it) == i && ok; + } + ok = cbm_qn_prefix_iter_next(&it) == -1 && cbm_qn_prefix_iter_next(&it) == -1 && ok; + if (visits) + *visits = it.visits; + return ok; +} + +static void pylsp_range_reset_failures(void) { + cbm_registry_test_type_qn_order_fail_after(-1); + cbm_registry_test_qn_prefix_block_fail_after(-1); + cbm_py_test_module_scratch_fail_after(-1); + cbm_py_test_module_prefix_fail_after(-1); +} + +TEST(pylsp_range_order_boundaries_and_null_contract) { + static const char *const qns[] = { + "p.foo.B", "p.foo.A.child", NULL, "p.foo.A-b", "p.foobar.C", "p.foo.AB", + "p.foo.A.z", "p.foo.", "", "p.foo.B", "p.foo.C.child", "p.foo.C-plain"}; + static const char *const prefixes[] = {NULL, "", "p.foo.", "p.foo", "p.foo.A", + "p.foo.A.", "p.foo.B", "p.foo..", "zzzz."}; + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg, empty; + cbm_registry_init(®, &arena); + cbm_registry_init(&empty, &arena); + bool ok = true; + for (size_t i = 0; i < sizeof(qns) / sizeof(qns[0]); i++) + ok = pylsp_range_add_type(®, qns[i], "Shared") && + pylsp_prefix_add(®, qns[i], i == 1 ? "Receiver" : NULL) && ok; + for (int indexed = 0; indexed < 2; indexed++) { + if (indexed) { + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.type_qn_sorted && reg.func_qn_sorted && + reg.type_qn_sorted_count == reg.type_count - 1 && + reg.type_qn_sorted_upto == reg.type_count && ok; + if (reg.type_qn_sorted) { + for (int i = 0; i < reg.type_qn_sorted_count; i++) { + const CBMRegistryTypeQNEntry *e = ®.type_qn_sorted[i]; + ok = e->type_index >= 0 && e->type_index < reg.type_count && + e->qualified_name == reg.types[e->type_index].qualified_name && ok; + } + } + } + for (int types = 0; types < 2; types++) { + for (int direct = 0; direct < 2; direct++) { + for (size_t p = 0; p < sizeof(prefixes) / sizeof(prefixes[0]); p++) { + CBMArena scratch; + cbm_arena_init(&scratch); + ok = pylsp_range_oracle(®, types != 0, prefixes[p], direct != 0, &scratch, + NULL) && + pylsp_range_oracle(®, types != 0, prefixes[p], direct != 0, NULL, + NULL) && + pylsp_range_oracle(&empty, types != 0, prefixes[p], direct != 0, &scratch, + NULL) && + pylsp_range_oracle(NULL, types != 0, prefixes[p], direct != 0, &scratch, + NULL) && + ok; + cbm_arena_destroy(&scratch); + } + } + } + } + ok = !cbm_registry_types_with_qn_prefix(®, "p.", true, &arena, NULL) && + !cbm_registry_funcs_with_qn_prefix(®, "p.", false, &arena, NULL) && + cbm_qn_prefix_iter_next(NULL) == -1 && + cbm_registry_has_func_qn_prefix(®, "p.foo.", NULL) && ok; + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_growth_rebuild_seal_and_independent_failure) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg, sealed; + cbm_registry_init(®, &arena); + cbm_registry_init(&sealed, &arena); + bool ok = pylsp_range_add_type(®, "p.old", "Old") && pylsp_prefix_add(®, "p.old", NULL); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + const CBMRegisteredType *old_types = reg.types; + const CBMRegisteredFunc *old_funcs = reg.funcs; + for (int i = 0; i < 140; i++) { + const char *qn = cbm_arena_sprintf(&arena, "p.tail%03d", i); + ok = qn && pylsp_range_add_type(®, qn, "Tail") && pylsp_prefix_add(®, qn, NULL) && ok; + } + ok = pylsp_range_add_type(®, NULL, "NoQN") && pylsp_prefix_add(®, NULL, NULL) && + reg.type_count == 142 && reg.func_count == 142 && reg.types != old_types && + reg.funcs != old_funcs && reg.type_qn_sorted_upto == 1 && reg.func_qn_sorted_upto == 1 && + pylsp_range_oracle(®, true, "p.", true, &arena, NULL) && + pylsp_range_oracle(®, false, "p.", true, &arena, NULL) && ok; + if (reg.type_count == 142 && reg.func_count == 142) { + /* The previous queries ended before either registered identity changes. */ + reg.types[0].qualified_name = "p.renamed"; + reg.funcs[0].qualified_name = "p.renamed"; + cbm_registry_build_func_qn_order(®, &arena); + const CBMRegistryQNEntry *func_order = reg.func_qn_sorted; + cbm_registry_test_type_qn_order_fail_after(0); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_test_type_qn_order_fail_after(-1); + ok = func_order && reg.func_qn_sorted == func_order && !reg.type_qn_sorted && + reg.type_qn_sorted_count == 0 && reg.type_qn_sorted_upto == 0 && + pylsp_range_oracle(®, true, "p.", true, &arena, NULL) && ok; + cbm_registry_test_type_qn_order_fail_after(1); + cbm_registry_build_type_qn_order(®, &arena); + bool first_built = reg.type_qn_sorted != NULL; + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_test_type_qn_order_fail_after(-1); + ok = first_built && !reg.type_qn_sorted && reg.func_qn_sorted == func_order && ok; + cbm_registry_build_type_qn_order(®, &arena); + const CBMRegistryTypeQNEntry *type_order = reg.type_qn_sorted; + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + ok = type_order && reg.type_qn_sorted == type_order && !reg.func_qn_sorted && + pylsp_range_oracle(®, false, "p.", true, &arena, NULL) && ok; + cbm_registry_build_func_qn_order(®, &arena); + func_order = reg.func_qn_sorted; + reg.read_only = true; + cbm_registry_test_type_qn_order_fail_after(0); + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + pylsp_range_reset_failures(); + cbm_registry_test_qn_order_fail_after(-1); + cbm_registry_add_type(®, (CBMRegisteredType){.qualified_name = "forbidden.Type"}); + cbm_registry_add_func(®, (CBMRegisteredFunc){.qualified_name = "forbidden.call"}); + ok = reg.type_qn_sorted == type_order && reg.func_qn_sorted == func_order && + reg.type_count == 142 && reg.func_count == 142 && + pylsp_range_oracle(®, true, "p.", true, &arena, NULL) && + pylsp_range_oracle(®, false, "p.", true, &arena, NULL) && ok; + } + ok = pylsp_range_add_type(&sealed, "p.sealed", "Sealed") && + pylsp_prefix_add(&sealed, "p.sealed", NULL) && ok; + sealed.read_only = true; + cbm_registry_build_type_qn_order(&sealed, &arena); + cbm_registry_build_func_qn_order(&sealed, &arena); + ok = !sealed.type_qn_sorted && !sealed.func_qn_sorted && + pylsp_range_oracle(&sealed, true, "p.", true, &arena, NULL) && + pylsp_range_oracle(&sealed, false, "p.", true, &arena, NULL) && ok; + pylsp_range_reset_failures(); + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_invalid_snapshot_uses_complete_scan) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = pylsp_range_add_type(®, "p.Z", "Z") && pylsp_range_add_type(®, "p.A", "A") && + pylsp_prefix_add(®, "p.Z", NULL) && pylsp_prefix_add(®, "p.A", NULL); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.type_count == 2 && reg.func_count == 2 && reg.type_qn_sorted && reg.func_qn_sorted && + ok; + size_t allocated = cbm_arena_total(&arena); + for (int types = 0; types < 2; types++) { + int *covered = types ? ®.type_qn_sorted_upto : ®.func_qn_sorted_upto; + int *count = types ? ®.type_qn_sorted_count : ®.func_qn_sorted_count; + int saved_covered = *covered, saved_count = *count; + for (int invalid = 0; invalid < 3; invalid++) { + *covered = invalid == 0 ? 3 : saved_covered; + *count = invalid == 1 ? -1 : (invalid == 2 ? 3 : saved_count); + int expected_covered = *covered, expected_count = *count; + uint64_t visits = 0; + ok = pylsp_range_oracle(®, types != 0, "p.", true, &arena, &visits) && visits >= 2 && + *covered == expected_covered && *count == expected_count && + cbm_arena_total(&arena) == allocated && ok; + *covered = saved_covered; + *count = saved_count; + } + ok = pylsp_range_oracle(®, types != 0, "p.", true, &arena, NULL) && ok; + allocated = cbm_arena_total(&arena); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_block_failure_preserves_each_link_and_tail) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry links[3]; + bool ok = true; + for (int j = 0; j < 3; j++) { + cbm_registry_init(&links[j], &arena); + ok = pylsp_range_add_type(&links[j], "p.Z", "Z") && + pylsp_range_add_type(&links[j], "p.A", "A") && + pylsp_prefix_add(&links[j], "p.Z", NULL) && pylsp_prefix_add(&links[j], "p.A", NULL) && + ok; + cbm_registry_build_type_qn_order(&links[j], &arena); + cbm_registry_build_func_qn_order(&links[j], &arena); + ok = links[j].type_qn_sorted && links[j].func_qn_sorted && + pylsp_range_add_type(&links[j], "p.Tail", "Tail") && + pylsp_prefix_add(&links[j], "p.Tail", NULL) && ok; + links[j].fallback = j < 2 ? &links[j + 1] : NULL; + } + for (int types = 0; types < 2; types++) { + for (int fail = 0; fail < 3; fail++) { + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_registry_test_qn_prefix_block_fail_after(fail); + for (int j = 0; j < 3; j++) { + uint64_t visits = 0; + ok = pylsp_range_oracle(&links[j], types != 0, "p.", true, &scratch, &visits) && + visits > 0 && ok; + } + cbm_registry_test_qn_prefix_block_fail_after(-1); + cbm_arena_destroy(&scratch); + } + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_direct_subtree_pruning_has_absolute_bound) { + enum { NESTED = 1024, BOUND = 100 }; + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = true; + for (int i = NESTED - 1; i >= 0; i--) { + const char *qn = cbm_arena_sprintf(&arena, "p.foo.A.n%04d", i); + ok = qn && pylsp_range_add_type(®, qn, "Nested") && + pylsp_prefix_add(®, qn, "p.foo.A") && ok; + } + static const char *const direct[] = {"p.foo.B", "p.foo.A-b", "p.foo.AB"}; + for (size_t i = 0; i < sizeof(direct) / sizeof(direct[0]); i++) + ok = pylsp_range_add_type(®, direct[i], "Direct") && + pylsp_prefix_add(®, direct[i], NULL) && ok; + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.type_count == NESTED + 3 && reg.func_count == NESTED + 3 && reg.type_qn_sorted && + reg.func_qn_sorted && ok; + for (int types = 0; types < 2; types++) { + uint64_t indexed = 0, linear = 0, miss = 0; + ok = pylsp_range_oracle(®, types != 0, "p.foo.", true, &arena, &indexed) && + indexed > 0 && indexed <= BOUND && + pylsp_range_oracle(®, types != 0, "zzzz.", true, &arena, &miss) && miss > 0 && + miss <= BOUND && ok; + cbm_registry_test_qn_prefix_block_fail_after(0); + ok = pylsp_range_oracle(®, types != 0, "p.foo.", true, &arena, &linear) && + linear >= NESTED + 3 && linear > BOUND && ok; + cbm_registry_test_qn_prefix_block_fail_after(-1); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +/* Test scratch storage must be gone before these retained bindings are read. */ +TEST(pylsp_range_class_binding_keeps_order_shadow_and_package_filters) { + static const char source[] = "class Local:\n pass\n"; + TSParser *parser = ts_parser_new(); + bool language = parser && ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + language ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool ok = tree && !ts_node_has_error(ts_tree_root_node(tree)); + for (int package = 0; tree && package < 2; package++) { + for (int failure = 0; failure < 6; failure++) { + CBMArena owner, arena; + cbm_arena_init(&owner); + cbm_arena_init(&arena); + CBMTypeRegistry head, middle, tail; + cbm_registry_init(&head, &owner); + cbm_registry_init(&middle, &owner); + cbm_registry_init(&tail, &owner); + head.fallback = &middle; + middle.fallback = &tail; + bool built = pylsp_range_add_type(&head, "test.pkg.Head", "Shared") && + pylsp_range_add_type(&head, "test.pkg.Same", "HeadSame") && + pylsp_range_add_type(&head, "test.pkg.Local", "Local") && + pylsp_range_add_type(&middle, "test.pkg.Middle", "Shared") && + pylsp_range_add_type(&middle, "test.pkg.Same", "Masked") && + pylsp_range_add_type(&middle, "test.pkg.Deep", "MiddleOnly") && + pylsp_range_add_type(&tail, "test.pkg.Tail", "Shared") && + pylsp_range_add_type(&tail, "test.pkg.Deep", "TailOnly") && + pylsp_range_add_type(&tail, "test.pkg.sub.Hidden", "Hidden"); + if (failure == 5) + cbm_registry_test_type_qn_order_fail_after(0); + CBMTypeRegistry *links[] = {&head, &middle, &tail}; + for (int i = 0; i < 3; i++) + cbm_registry_build_type_qn_order(links[i], &owner); + cbm_registry_test_type_qn_order_fail_after(-1); + if (failure != 5) + built = + head.type_qn_sorted && middle.type_qn_sorted && tail.type_qn_sorted && built; + else + built = + !head.type_qn_sorted && !middle.type_qn_sorted && !tail.type_qn_sorted && built; + /* The two fallback Deep entries remain visible: only the head shadows. */ + bool shadowed = built && cbm_registry_type_shadowed(&head, &middle, 1) && + !cbm_registry_type_shadowed(&head, &tail, 1) && + !cbm_registry_type_shadowed(&head, &head, 1); + PyLSPContext ctx; + CBMResolvedCallArray out = {0}; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &head, + package ? "test.pkg.__init__" : "test.pkg", &out); + if (failure == 1 || failure == 2) + cbm_registry_test_qn_prefix_block_fail_after(failure - 1); + if (failure == 3) + cbm_py_test_module_prefix_fail_after(0); + if (failure == 4) + cbm_py_test_module_scratch_fail_after(0); + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + pylsp_range_reset_failures(); + ok = shadowed && !ctx.callable_value_proof_disabled && + ctx.eval_failure == CBM_LSP_COMPLETE && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "Shared"), + "test.pkg.Tail") && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "HeadSame"), + "test.pkg.Same") && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "MiddleOnly"), + "test.pkg.Deep") && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "TailOnly"), + "test.pkg.Deep") && + !cbm_scope_lookup_local(ctx.current_scope, "Masked") && + !cbm_scope_lookup_local(ctx.current_scope, "Local") && ok; + const CBMType *hidden = cbm_scope_lookup_local(ctx.current_scope, "Hidden"); + ok = + (package ? hidden == NULL : pylsp_init_named_type(hidden, "test.pkg.sub.Hidden")) && + ok; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&owner); + } + } + pylsp_range_reset_failures(); + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_wildcard_and_scratch_failures_keep_exact_later_calls) { + static const char source[] = "external = External()\nexternal.ping()\n" + "from vendor import *\nshadow_only(72)\nExternal(74)\n" + "def later(v):\n return v\nlater(73)\n"; + CBMFileResult *raw = extract_py(source); + TSParser *parser = ts_parser_new(); + bool language = parser && ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + language ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool healthy = raw && !raw->has_error && !raw->parse_incomplete && !raw->lsp_skipped && tree && + !ts_node_has_error(ts_tree_root_node(tree)) && + pylsp_module_has_def(raw, "test.main.later"); + bool ok = healthy; + for (int failure = 0; healthy && failure < 8; failure++) { + CBMArena owner, arena; + cbm_arena_init(&owner); + cbm_arena_init(&arena); + CBMTypeRegistry head, tail; + cbm_registry_init(&head, &owner); + cbm_registry_init(&tail, &owner); + head.fallback = &tail; + bool built = pylsp_range_add_type(&head, "test.main.External", "External") && + pylsp_range_add_type(&tail, "test.main.Other", "Other"); + cbm_registry_add_func(&head, + (CBMRegisteredFunc){.qualified_name = "test.main.External.ping", + .short_name = "ping", + .receiver_type = "test.main.External"}); + cbm_registry_add_func( + &head, (CBMRegisteredFunc){.qualified_name = "test.main.later", .short_name = "later"}); + /* A head method must not suppress invalidation of a fallback free function. */ + cbm_registry_add_func(&head, (CBMRegisteredFunc){.qualified_name = "test.main.hidden", + .short_name = "hidden", + .receiver_type = "test.main.External"}); + cbm_registry_add_func(&tail, (CBMRegisteredFunc){.qualified_name = "test.main.hidden", + .short_name = "shadow_only"}); + cbm_registry_build_type_qn_order(&head, &owner); + cbm_registry_build_type_qn_order(&tail, &owner); + cbm_registry_build_func_qn_order(&head, &owner); + cbm_registry_build_func_qn_order(&tail, &owner); + built = built && head.func_count == 3 && tail.func_count == 1 && head.type_qn_sorted && + tail.type_qn_sorted && head.func_qn_sorted && tail.func_qn_sorted; + head.read_only = true; + tail.read_only = true; + PyLSPContext ctx; + CBMResolvedCallArray out = {0}; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &head, "test.main", &out); + if (failure >= 1 && failure <= 4) + cbm_registry_test_qn_prefix_block_fail_after(failure - 1); + if (failure == 5) + cbm_py_test_module_prefix_fail_after(0); + if (failure == 6) + cbm_py_test_module_scratch_fail_after(0); + if (failure == 7) + cbm_py_test_module_prefix_fail_after(1); + uint64_t before = + atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed); + py_lsp_process_file(&ctx, ts_tree_root_node(tree)); + pylsp_range_reset_failures(); + const CBMType *shadow = cbm_scope_lookup_local(ctx.current_scope, "shadow_only"); + ok = built && ctx.eval_failure == CBM_LSP_COMPLETE && !ctx.callable_value_proof_disabled && + shadow && cbm_type_is_unknown(shadow) && + atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed) > before && + pylsp_module_site_joins(&raw->calls, &out, source, "external.ping()", "test.main", + "test.main.External.ping") && + pylsp_module_site_joins(&raw->calls, &out, source, "later(73)", "test.main", + "test.main.later") && + pylsp_prefix_site_unresolved(&raw->calls, &out, source, "shadow_only(72)", + "test.main") && + pylsp_prefix_site_unresolved(&raw->calls, &out, source, "External(74)", "test.main") && + ok; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&owner); + } + pylsp_range_reset_failures(); + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + if (raw) + cbm_free_result(raw); + ASSERT_TRUE(ok); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_range_order_boundaries_and_null_contract); + RUN_TEST(pylsp_range_growth_rebuild_seal_and_independent_failure); + RUN_TEST(pylsp_range_invalid_snapshot_uses_complete_scan); + RUN_TEST(pylsp_range_block_failure_preserves_each_link_and_tail); + RUN_TEST(pylsp_range_direct_subtree_pruning_has_absolute_bound); + RUN_TEST(pylsp_range_class_binding_keeps_order_shadow_and_package_filters); + RUN_TEST(pylsp_range_wildcard_and_scratch_failures_keep_exact_later_calls); + RUN_TEST(pylsp_prefix_order_matches_iterator_and_chain_shadowing); RUN_TEST(pylsp_prefix_order_survives_array_growth_rebuild_and_seal); RUN_TEST(pylsp_prefix_order_allocation_failure_preserves_oracle);