diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 025ea1966..bd0664ef6 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -114,6 +114,190 @@ void cbm_reset_profile(void) { atomic_store(&total_files, 0); } +/* New facts use checked allocation and atomic publication; legacy extraction + * arrays retain their existing contracts. */ +static bool py_namespace_nonempty(const char *s) { + return s && s[0]; +} + +static bool py_namespace_fact_valid(const CBMPyNamespaceFact *f) { + if (!f || f->name_count < 0 || (size_t)f->name_count > SIZE_MAX / sizeof(*f->names) || + (f->name_count == 0 ? f->names != NULL : f->names == NULL)) + return false; + bool literal = f->kind == CBM_PY_NS_ALL_SET || f->kind == CBM_PY_NS_ALL_APPEND; + if (!literal && (f->names || f->name_count || f->sequence_kind != CBM_PY_NS_SEQUENCE_NONE)) + return false; + if (f->kind != CBM_PY_NS_OWN_DEF && f->def_kind != CBM_PY_NS_DEF_NONE) + return false; + if (f->kind != CBM_PY_NS_IMPORT_MODULE && f->flags) + return false; + if (f->kind != CBM_PY_NS_IMPORT_NAME && f->kind != CBM_PY_NS_IMPORT_STAR && f->relative_level) + return false; + bool unknown = f->kind == CBM_PY_NS_SHADOW || f->kind == CBM_PY_NS_UNKNOWN || + f->kind == CBM_PY_NS_ALL_UNKNOWN; + if (!unknown && f->reason != CBM_PY_NS_REASON_NONE) + return false; + switch (f->kind) { + case CBM_PY_NS_OWN_DEF: + return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name && + (f->def_kind == CBM_PY_NS_DEF_FUNCTION || f->def_kind == CBM_PY_NS_DEF_CLASS); + case CBM_PY_NS_IMPORT_MODULE: + return py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->module_name) && + !f->member_name && !(f->flags & ~CBM_PY_NS_IMPORT_BINDS_ROOT); + case CBM_PY_NS_IMPORT_NAME: + case CBM_PY_NS_IMPORT_STAR: + if (!f->module_name || (!f->module_name[0] && !f->relative_level)) + return false; + return f->kind == CBM_PY_NS_IMPORT_NAME + ? py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->member_name) + : !f->local_name && !f->member_name; + case CBM_PY_NS_SHADOW: + return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_DECORATED || + f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED); + case CBM_PY_NS_DELETE: + return py_namespace_nonempty(f->local_name) && strcmp(f->local_name, "__all__") != 0 && + !f->module_name && !f->member_name; + case CBM_PY_NS_UNKNOWN: + return !f->local_name && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_COMPOUND || f->reason == CBM_PY_NS_REASON_EFFECT || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE); + case CBM_PY_NS_ALL_UNKNOWN: + return !f->local_name && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_EFFECT || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE); + case CBM_PY_NS_ALL_SET: + case CBM_PY_NS_ALL_APPEND: + if (f->local_name || f->module_name || f->member_name || + (f->sequence_kind != CBM_PY_NS_SEQUENCE_LIST && + f->sequence_kind != CBM_PY_NS_SEQUENCE_TUPLE)) + return false; + for (int i = 0; i < f->name_count; i++) { + if (!f->names[i]) + return false; + } + return true; + case CBM_PY_NS_ALL_DELETE: + return !f->local_name && !f->module_name && !f->member_name; + default: + return false; + } +} + +bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *f) { + if (!f) + return false; + bool empty = !f->items && f->count == 0 && f->cap == 0; + if (f->version == 0) + return f->status == CBM_PY_NS_NOT_CAPTURED && f->failure == CBM_PY_NS_FAILURE_NONE && empty; + if (f->version != CBM_PY_NAMESPACE_FACTS_VERSION || f->language < 0 || + f->language >= CBM_LANG_COUNT) + return false; + switch (f->status) { + case CBM_PY_NS_NOT_CAPTURED: + return empty && f->failure == CBM_PY_NS_FAILURE_NONE; + case CBM_PY_NS_NOT_APPLICABLE: + return empty && f->language != CBM_LANG_PYTHON && f->failure == CBM_PY_NS_FAILURE_NONE; + case CBM_PY_NS_INCOMPLETE: + return empty && f->failure >= CBM_PY_NS_FAILURE_INVALID_INPUT && + f->failure <= CBM_PY_NS_FAILURE_LIMIT; + case CBM_PY_NS_COMPLETE: + if (f->language != CBM_LANG_PYTHON || f->failure != CBM_PY_NS_FAILURE_NONE || + f->count < 0 || f->cap < f->count || (size_t)f->cap > SIZE_MAX / sizeof(*f->items) || + (f->count == 0 ? !empty : f->items == NULL)) + return false; + for (int i = 0; i < f->count; i++) { + if (!py_namespace_fact_valid(&f->items[i])) + return false; + } + return true; + default: + return false; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static CBM_TLS int py_namespace_copy_left = -1; +void cbm_py_namespace_test_copy_fail_after(int successful_allocations) { + py_namespace_copy_left = successful_allocations; +} +#endif + +static void *py_namespace_copy_alloc(CBMArena *arena, size_t size) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_namespace_copy_left == 0) + return NULL; + if (py_namespace_copy_left > 0) + py_namespace_copy_left--; +#endif + return cbm_arena_alloc(arena, size); +} + +static bool py_namespace_copy_string(CBMArena *arena, const char *source, const char **out) { + *out = NULL; + if (!source) + return true; + size_t len = strlen(source); + if (len == SIZE_MAX) + return false; + char *copy = py_namespace_copy_alloc(arena, len + 1); + if (!copy) + return false; + memcpy(copy, source, len + 1); + *out = copy; + return true; +} + +bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source, + CBMPyNamespaceFacts *out) { + if (!out || source == out) + return false; + CBMPyNamespaceFacts failure = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_INCOMPLETE, + .failure = CBM_PY_NS_FAILURE_INVALID_INPUT}; + if (!arena || !cbm_py_namespace_facts_valid(source)) { + *out = failure; + return false; + } + failure.language = source->language; + failure.failure = CBM_PY_NS_FAILURE_ALLOCATION; + CBMPyNamespaceFacts copy = *source; + copy.items = NULL; + copy.cap = copy.count; + if (copy.count) { + copy.items = py_namespace_copy_alloc(arena, (size_t)copy.count * sizeof(*copy.items)); + if (!copy.items) + goto failed; + memcpy(copy.items, source->items, (size_t)copy.count * sizeof(*copy.items)); + } + for (int i = 0; i < copy.count; i++) { + const CBMPyNamespaceFact *src = &source->items[i]; + CBMPyNamespaceFact *dst = ©.items[i]; + if (!py_namespace_copy_string(arena, src->local_name, &dst->local_name) || + !py_namespace_copy_string(arena, src->module_name, &dst->module_name) || + !py_namespace_copy_string(arena, src->member_name, &dst->member_name)) + goto failed; + dst->names = NULL; + if (src->name_count) { + dst->names = + py_namespace_copy_alloc(arena, (size_t)src->name_count * sizeof(*dst->names)); + if (!dst->names) + goto failed; + for (int j = 0; j < src->name_count; j++) { + if (!py_namespace_copy_string(arena, src->names[j], &dst->names[j])) + goto failed; + } + } + } + *out = copy; + return true; +failed: + *out = failure; + return false; +} + // --- Growable array push functions --- #define GROW_ARRAY(arr, arena) \ @@ -3007,6 +3191,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C return NULL; } + result->py_namespace.version = CBM_PY_NAMESPACE_FACTS_VERSION; + result->py_namespace.language = language; + result->py_namespace.status = + language == CBM_LANG_PYTHON ? CBM_PY_NS_NOT_CAPTURED : CBM_PY_NS_NOT_APPLICABLE; cbm_work_arena_take(&result->arena); CBMArena *a = &result->arena; if (!cbm_test_declarations_validate(result, test_declarations)) { @@ -3252,6 +3440,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_extract_definitions(&ctx); cbm_test_declarations_finish(&ctx); cbm_extract_imports(&ctx); + cbm_extract_python_namespace_facts(&ctx); cbm_extract_unified(&ctx); if (cbm_lang_needs_cpp_branch_views(language)) { extract_cpp_branch_views(&ctx, ts_lang, timeout_micros); @@ -4002,6 +4191,35 @@ CBMFileResult *cbm_extract_file_ex_with_tests( CBMFileResult *result = extract_file_ex_body( source, source_len, language, project, rel_path, timeout_micros, extra_defines, include_paths, macro_table, return_type_table, &scratch, test_declarations); + /* Even early parse/quarantine exits publish conservative Python metadata. + * No tree is reparsed, and existing best-effort extraction flags stay intact. */ + if (result && language == CBM_LANG_PYTHON && + result->py_namespace.status == CBM_PY_NS_NOT_CAPTURED) { + CBMPyNamespaceFailure failure = CBM_PY_NS_FAILURE_NONE; + if (!result->module_qn || !result->module_qn[0]) { + if (!project || !project[0] || !rel_path || !rel_path[0]) { + failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + } else { + result->module_qn = + cbm_fqn_module_source_lang(&result->arena, project, rel_path, language); + if (!result->module_qn || !result->module_qn[0]) + failure = CBM_PY_NS_FAILURE_ALLOCATION; + } + } + if (failure) { + result->py_namespace = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = language, + .status = CBM_PY_NS_INCOMPLETE, + .failure = failure}; + } else { + CBMExtractCtx facts_ctx = {.arena = &result->arena, + .result = result, + .source = source, + .source_len = source_len, + .language = language}; + cbm_extract_python_namespace_facts(&facts_ctx); + } + } /* !tl_scratch_live: a nested extraction (an embedded language inside this * file) may already have parked its own; never overwrite it. */ /* Kept up to CBM_EXTRACT_SCRATCH_KEEP_BYTES, grown blocks included: the diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index f9d42ec36..e200972d9 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -389,6 +389,79 @@ typedef struct { CBMImportKind kind; // CBM_IMPORT_KIND_DEFAULT unless the syntax names a member kind } CBMImport; +/* Content-only ordered Python namespace facts; no resolved target identities. */ +#define CBM_PY_NAMESPACE_FACTS_VERSION 1u + +typedef enum { + CBM_PY_NS_NOT_CAPTURED = 0, + CBM_PY_NS_NOT_APPLICABLE = 1, + CBM_PY_NS_COMPLETE = 2, + CBM_PY_NS_INCOMPLETE = 3 +} CBMPyNamespaceStatus; +typedef enum { + CBM_PY_NS_FAILURE_NONE = 0, + CBM_PY_NS_FAILURE_INVALID_INPUT = 1, + CBM_PY_NS_FAILURE_ALLOCATION = 2, + CBM_PY_NS_FAILURE_LIMIT = 3 +} CBMPyNamespaceFailure; +typedef enum { + CBM_PY_NS_OWN_DEF = 1, + CBM_PY_NS_IMPORT_MODULE = 2, + CBM_PY_NS_IMPORT_NAME = 3, + CBM_PY_NS_IMPORT_STAR = 4, + CBM_PY_NS_SHADOW = 5, + CBM_PY_NS_DELETE = 6, + CBM_PY_NS_UNKNOWN = 7, + CBM_PY_NS_ALL_SET = 8, + CBM_PY_NS_ALL_APPEND = 9, + CBM_PY_NS_ALL_UNKNOWN = 10, + CBM_PY_NS_ALL_DELETE = 11 +} CBMPyNamespaceFactKind; +typedef enum { + CBM_PY_NS_DEF_NONE = 0, + CBM_PY_NS_DEF_FUNCTION = 1, + CBM_PY_NS_DEF_CLASS = 2 +} CBMPyNamespaceDefKind; +typedef enum { + CBM_PY_NS_SEQUENCE_NONE = 0, + CBM_PY_NS_SEQUENCE_LIST = 1, + CBM_PY_NS_SEQUENCE_TUPLE = 2 +} CBMPyNamespaceSequenceKind; +typedef enum { + CBM_PY_NS_REASON_NONE = 0, + CBM_PY_NS_REASON_VALUE = 1, + CBM_PY_NS_REASON_DECORATED = 2, + CBM_PY_NS_REASON_UNPROVEN_CLASS = 3, + CBM_PY_NS_REASON_COMPOUND = 4, + CBM_PY_NS_REASON_EFFECT = 5, + CBM_PY_NS_REASON_UNSUPPORTED = 6, + CBM_PY_NS_REASON_PARSE = 7 +} CBMPyNamespaceUnknownReason; +enum { CBM_PY_NS_IMPORT_BINDS_ROOT = 1u << 0 }; + +typedef struct { + CBMPyNamespaceFactKind kind; + CBMPyNamespaceDefKind def_kind; + CBMPyNamespaceSequenceKind sequence_kind; + CBMPyNamespaceUnknownReason reason; + uint32_t flags; + uint32_t relative_level; + const char *local_name; + const char *module_name; + const char *member_name; + const char **names; + int name_count; +} CBMPyNamespaceFact; +typedef struct { + uint32_t version; + CBMLanguage language; + CBMPyNamespaceStatus status; + CBMPyNamespaceFailure failure; + CBMPyNamespaceFact *items; + int count; + int cap; +} CBMPyNamespaceFacts; + typedef enum { CBM_USAGE_VALUE = 0, CBM_USAGE_CALL_REFERENCE, @@ -728,6 +801,7 @@ typedef struct CBMFileResult { CBMDefArray defs; CBMCallArray calls; CBMImportArray imports; + CBMPyNamespaceFacts py_namespace; CBMUsageArray usages; CBMThrowArray throws; CBMRWArray rw; @@ -1186,6 +1260,15 @@ int cbm_macro_extraction_enabled(void); // --- Internal helpers used by extractors --- +CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx); +bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *facts); +bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source, + CBMPyNamespaceFacts *out); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_namespace_test_capture_fail_after(int successful_allocations); +void cbm_py_namespace_test_copy_fail_after(int successful_allocations); +#endif + // Growable array push functions (arena-allocated, no individual free needed). void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def); void cbm_calls_push(CBMCallArray *arr, CBMArena *a, CBMCall call); diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index f3d627cd0..ee5b9478f 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -4,8 +4,10 @@ #include "lang_specs.h" // CBMLangSpec, CBMEmbeddedLangSpec, cbm_lang_spec, cbm_ts_language #include "tree_sitter/api.h" // TSNode, ts_node_* #include "foundation/constants.h" +#include "foundation/compat.h" /* CBM_TLS */ #include "extract_node_stack.h" #include // uint32_t +#include #include #include @@ -341,6 +343,490 @@ static void parse_python_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } +/* Ordered Python namespace facts are a separate content-only pass. The older + * import records keep their existing graph/heuristic meaning. */ +typedef struct { + CBMExtractCtx *ctx; + CBMPyNamespaceFacts facts; + CBMPyNamespaceFailure failure; +} PyNamespaceBuilder; + +#ifdef CBM_ENABLE_TEST_SEAMS +static CBM_TLS int py_namespace_capture_left = -1; +void cbm_py_namespace_test_capture_fail_after(int successful_allocations) { + py_namespace_capture_left = successful_allocations; +} +#endif + +static void *py_ns_alloc(PyNamespaceBuilder *b, size_t size) { + if (b->failure) + return NULL; +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_namespace_capture_left == 0) { + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return NULL; + } + if (py_namespace_capture_left > 0) + py_namespace_capture_left--; +#endif + void *p = cbm_arena_alloc(b->ctx->arena, size); + if (!p) + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return p; +} + +static const char *py_ns_copy(PyNamespaceBuilder *b, const char *text, size_t len) { + if (len == SIZE_MAX) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return NULL; + } + char *copy = py_ns_alloc(b, len + 1); + if (!copy) + return NULL; + memcpy(copy, text, len); + copy[len] = '\0'; + return copy; +} + +static bool py_ns_slice(PyNamespaceBuilder *b, TSNode node, const char **text, size_t *len) { + if (ts_node_is_null(node)) + return false; + uint32_t start = ts_node_start_byte(node), end = ts_node_end_byte(node); + if (end < start || end > (uint32_t)b->ctx->source_len) + return false; + *text = b->ctx->source + start; + *len = (size_t)(end - start); + return memchr(*text, '\0', *len) == NULL; +} + +static bool py_ns_identifier_bytes(const char *s, size_t n, bool dotted) { + bool initial = true; + if (!n) + return false; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (dotted && c == '.' && !initial) { + initial = true; + continue; + } + if (c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (!initial && c >= '0' && c <= '9')) { + initial = false; + continue; + } + return false; + } + return !initial; +} + +static const char *py_ns_name(PyNamespaceBuilder *b, TSNode node, bool dotted) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(b, node, &text, &len) || !py_ns_identifier_bytes(text, len, dotted)) + return NULL; + return py_ns_copy(b, text, len); +} + +static bool py_ns_push(PyNamespaceBuilder *b, CBMPyNamespaceFact fact) { + if (b->failure) + return false; + if (b->facts.count == b->facts.cap) { + if (b->facts.cap > INT_MAX / 2) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + int cap = b->facts.cap ? b->facts.cap * 2 : 8; + if ((size_t)cap > SIZE_MAX / sizeof(*b->facts.items)) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + CBMPyNamespaceFact *items = py_ns_alloc(b, (size_t)cap * sizeof(*items)); + if (!items) + return false; + if (b->facts.count) + memcpy(items, b->facts.items, (size_t)b->facts.count * sizeof(*items)); + b->facts.items = items; + b->facts.cap = cap; + } + b->facts.items[b->facts.count++] = fact; + return true; +} + +static bool py_ns_unknown(PyNamespaceBuilder *b, CBMPyNamespaceUnknownReason reason) { + return py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_UNKNOWN, .reason = reason}); +} + +static bool py_ns_plain_string(PyNamespaceBuilder *b, TSNode node, const char **text, size_t *len) { + if (strcmp(ts_node_type(node), "string") != 0 || !py_ns_slice(b, node, text, len) || *len < 2) + return false; + char quote = (*text)[0]; + if ((quote != '\'' && quote != '"') || (*text)[*len - 1] != quote) + return false; + for (size_t i = 1; i + 1 < *len; i++) { + if ((*text)[i] == '\\' || (*text)[i] == quote || (*text)[i] == '\n' || (*text)[i] == '\r') + return false; + } + (*text)++; + *len -= 2; + return true; +} + +/* Bounded syntactic proof only. Unsupported or deep expressions become an + * unknown-effect barrier; no evaluator or source execution is involved. */ +static bool py_ns_safe_expr(PyNamespaceBuilder *b, TSNode node, int depth) { + if (ts_node_is_null(node)) + return true; + if (depth >= 64) + return false; + const char *kind = ts_node_type(node); + if (strcmp(kind, "identifier") == 0 || strcmp(kind, "integer") == 0 || + strcmp(kind, "float") == 0 || strcmp(kind, "true") == 0 || strcmp(kind, "false") == 0 || + strcmp(kind, "none") == 0 || strcmp(kind, "comment") == 0) + return true; + if (strcmp(kind, "string") == 0) { + const char *text = NULL; + size_t len = 0; + return py_ns_plain_string(b, node, &text, &len); + } + bool container = + strcmp(kind, "tuple") == 0 || strcmp(kind, "list") == 0 || + strcmp(kind, "parenthesized_expression") == 0 || strcmp(kind, "parameters") == 0 || + strcmp(kind, "default_parameter") == 0 || strcmp(kind, "typed_parameter") == 0 || + strcmp(kind, "typed_default_parameter") == 0 || strcmp(kind, "list_splat_pattern") == 0 || + strcmp(kind, "dictionary_splat_pattern") == 0 || strcmp(kind, "type") == 0; + if (!container) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool safe = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_is_named(child) && !py_ns_safe_expr(b, child, depth + 1)) { + safe = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return safe; +} + +static bool py_ns_signature_safe(PyNamespaceBuilder *b, TSNode node) { + return py_ns_safe_expr(b, ts_node_child_by_field_name(node, TS_FIELD("parameters")), 0) && + py_ns_safe_expr(b, ts_node_child_by_field_name(node, TS_FIELD("return_type")), 0); +} + +static bool py_ns_class_safe(PyNamespaceBuilder *b, TSNode node) { + if (!ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("superclasses")))) + return false; + TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body")); + if (ts_node_is_null(body)) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool safe = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child)) + continue; + const char *kind = ts_node_type(child); + if (strcmp(kind, "pass_statement") == 0 || strcmp(kind, "comment") == 0) + continue; + if (strcmp(kind, "function_definition") == 0 && py_ns_signature_safe(b, child)) + continue; + if (strcmp(kind, "expression_statement") == 0 && + ts_node_named_child_count(child) == 1) { + TSNode expr = ts_node_named_child(child, 0); + if (strcmp(ts_node_type(expr), "string") == 0 && py_ns_safe_expr(b, expr, 0)) + continue; + /* Even an identifier assignment can install a descriptor whose + * __set_name__ hook executes during class creation. */ + } + safe = false; + break; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return safe; +} + +static bool py_ns_import(PyNamespaceBuilder *b, TSNode node, bool from) { + TSNode module = from ? ts_node_child_by_field_name(node, TS_FIELD("module_name")) : (TSNode){0}; + const char *module_name = NULL; + uint32_t level = 0; + if (from) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(b, module, &text, &len)) + return false; + while ((size_t)level < len && text[level] == '.') + level++; + if (level == len ? level == 0 : !py_ns_identifier_bytes(text + level, len - level, true)) + return false; + module_name = py_ns_copy(b, text + level, len - level); + if (!module_name) + return false; + } + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + int emitted = 0; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || (from && child.id == module.id) || + strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *kind = ts_node_type(child); + CBMPyNamespaceFact f = {.kind = from ? CBM_PY_NS_IMPORT_NAME : CBM_PY_NS_IMPORT_MODULE, + .module_name = module_name, + .relative_level = level}; + if (from && strcmp(kind, "wildcard_import") == 0) { + f.kind = CBM_PY_NS_IMPORT_STAR; + } else { + bool alias = strcmp(kind, "aliased_import") == 0; + TSNode name = alias ? ts_node_child_by_field_name(child, TS_FIELD("name")) : child; + if (!alias && strcmp(kind, "dotted_name") != 0 && strcmp(kind, "identifier") != 0) { + ok = false; + break; + } + const char *imported = py_ns_name(b, name, !from); + if (!imported) { + ok = false; + break; + } + if (from) + f.member_name = imported; + else + f.module_name = imported; + if (alias) { + f.local_name = + py_ns_name(b, ts_node_child_by_field_name(child, TS_FIELD("alias")), false); + } else if (from) { + f.local_name = imported; + } else { + size_t n = strcspn(imported, "."); + f.local_name = py_ns_copy(b, imported, n); + f.flags = CBM_PY_NS_IMPORT_BINDS_ROOT; + } + if (!f.local_name) { + ok = false; + break; + } + } + if (!py_ns_push(b, f)) { + ok = false; + break; + } + emitted++; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok && emitted > 0; +} + +static bool py_ns_all_literal(PyNamespaceBuilder *b, TSNode node, CBMPyNamespaceFact *fact) { + if (ts_node_is_null(node)) + return false; + const char *kind = ts_node_type(node); + if (strcmp(kind, "list") != 0 && strcmp(kind, "tuple") != 0) + return false; + uint32_t count = ts_node_named_child_count(node); + if (count > INT_MAX || (size_t)count > SIZE_MAX / sizeof(*fact->names)) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + const char **names = count ? py_ns_alloc(b, (size_t)count * sizeof(*names)) : NULL; + if (count && !names) + return false; + int used = 0; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *text = NULL; + size_t len = 0; + if (!py_ns_plain_string(b, child, &text, &len) || + !(names[used] = py_ns_copy(b, text, len))) { + ok = false; + break; + } + used++; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + if (!ok) + return false; + fact->sequence_kind = + strcmp(kind, "list") == 0 ? CBM_PY_NS_SEQUENCE_LIST : CBM_PY_NS_SEQUENCE_TUPLE; + fact->names = used ? names : NULL; + fact->name_count = used; + return true; +} + +static bool py_ns_assignment(PyNamespaceBuilder *b, TSNode node) { + TSNode left = ts_node_child_by_field_name(node, TS_FIELD("left")); + TSNode right = ts_node_child_by_field_name(node, TS_FIELD("right")); + TSNode annotation = ts_node_child_by_field_name(node, TS_FIELD("type")); + const char *name = py_ns_name(b, left, false); + if (!name) + return false; + if (!py_ns_safe_expr(b, right, 0) || !py_ns_safe_expr(b, annotation, 0)) { + if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) + return false; + } + if (ts_node_is_null(right)) + return true; /* annotation alone does not bind the target */ + bool augmented = strcmp(ts_node_type(node), "augmented_assignment") == 0; + if (strcmp(name, "__all__") != 0) { + if (augmented) + return py_ns_unknown(b, CBM_PY_NS_REASON_UNSUPPORTED); + return py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = CBM_PY_NS_REASON_VALUE}); + } + CBMPyNamespaceFact fact = {.kind = augmented ? CBM_PY_NS_ALL_APPEND : CBM_PY_NS_ALL_SET}; + bool plus = true; + if (augmented) { + const char *op = NULL; + size_t len = 0; + plus = py_ns_slice(b, ts_node_child_by_field_name(node, TS_FIELD("operator")), &op, &len) && + len == 2 && memcmp(op, "+=", 2) == 0; + } + if (plus && py_ns_all_literal(b, right, &fact)) + return py_ns_push(b, fact); + return py_ns_push( + b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_ALL_UNKNOWN, .reason = CBM_PY_NS_REASON_VALUE}); +} + +static bool py_ns_delete(PyNamespaceBuilder *b, TSNode node, int depth) { + if (depth > 2) + return false; + if (strcmp(ts_node_type(node), "identifier") == 0) { + const char *name = py_ns_name(b, node, false); + if (!name) + return false; + return py_ns_push(b, + strcmp(name, "__all__") == 0 + ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_ALL_DELETE} + : (CBMPyNamespaceFact){.kind = CBM_PY_NS_DELETE, .local_name = name}); + } + const char *kind = ts_node_type(node); + if (strcmp(kind, "delete_statement") != 0 && strcmp(kind, "expression_list") != 0) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_is_named(child) && strcmp(ts_node_type(child), "comment") != 0 && + !py_ns_delete(b, child, depth + 1)) { + ok = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok; +} + +static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { + const char *kind = ts_node_type(node); + if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) + return true; + if (strcmp(kind, "import_statement") == 0) + return py_ns_import(b, node, false); + if (strcmp(kind, "import_from_statement") == 0) + return py_ns_import(b, node, true); + bool decorated = strcmp(kind, "decorated_definition") == 0; + TSNode def = decorated ? ts_node_child_by_field_name(node, TS_FIELD("definition")) : node; + if (!ts_node_is_null(def)) { + const char *dk = ts_node_type(def); + bool function = strcmp(dk, "function_definition") == 0; + bool class_def = strcmp(dk, "class_definition") == 0; + if (function || class_def) { + const char *name = + py_ns_name(b, ts_node_child_by_field_name(def, TS_FIELD("name")), false); + if (!name) + return false; + bool proven = !decorated && (function || py_ns_class_safe(b, def)); + if (!proven || (function && !py_ns_signature_safe(b, def))) { + if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) + return false; + } + return py_ns_push( + b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, + .local_name = name, + .def_kind = function ? CBM_PY_NS_DEF_FUNCTION + : CBM_PY_NS_DEF_CLASS} + : (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = decorated + ? CBM_PY_NS_REASON_DECORATED + : CBM_PY_NS_REASON_UNPROVEN_CLASS}); + } + } + if (strcmp(kind, "delete_statement") == 0) + return py_ns_delete(b, node, 0); + if (strcmp(kind, "expression_statement") == 0 && ts_node_named_child_count(node) == 1) { + TSNode expr = ts_node_named_child(node, 0); + const char *ek = ts_node_type(expr); + if (strcmp(ek, "assignment") == 0 || strcmp(ek, "augmented_assignment") == 0) + return py_ns_assignment(b, expr); + return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); + } + if (strcmp(kind, "if_statement") == 0 || strcmp(kind, "try_statement") == 0 || + strcmp(kind, "for_statement") == 0 || strcmp(kind, "while_statement") == 0 || + strcmp(kind, "with_statement") == 0 || strcmp(kind, "match_statement") == 0) + return py_ns_unknown(b, CBM_PY_NS_REASON_COMPOUND); + return false; +} + +CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result) + return CBM_PY_NS_INCOMPLETE; + PyNamespaceBuilder b = {.ctx = ctx, + .facts = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = ctx->language, + .status = CBM_PY_NS_COMPLETE}}; + if (ctx->language != CBM_LANG_PYTHON) { + b.facts.status = CBM_PY_NS_NOT_APPLICABLE; + } else if (!ctx->arena || !ctx->source || ctx->source_len < 0) { + b.failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + } else if (ts_node_is_null(ctx->root) || ts_node_has_error(ctx->root)) { + py_ns_unknown(&b, CBM_PY_NS_REASON_PARSE); + } else { + TSTreeCursor cursor = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode node = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(node)) + continue; + int start = b.facts.count; + if (!py_ns_statement(&b, node) && !b.failure) { + b.facts.count = start; + py_ns_unknown(&b, CBM_PY_NS_REASON_UNSUPPORTED); + } + if (b.failure) + break; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + } + if (!b.failure && !cbm_py_namespace_facts_valid(&b.facts)) + b.failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + if (b.failure) { + b.facts = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = ctx->language, + .status = CBM_PY_NS_INCOMPLETE, + .failure = b.failure}; + } + ctx->result->py_namespace = b.facts; + return b.facts.status; +} + // --- ES module imports (JS/TS/TSX) --- // import X from "Y"; import {A, B} from "Y"; import * as X from "Y" // const X = require("Y") diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index b9e4d2be6..6007e7d1e 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -302,6 +302,54 @@ static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { } } +/* Facts have counted lists, not sentinel arrays. Bail before touching a failed + * array copy so no string rewrite can land in the original result's storage. */ +static bool cr_py_size(cr_ctx_t *c, size_t size) { + if (c->failed) + return false; + if (c->phase == CR_MEASURE && + (size > SIZE_MAX - CR_ALIGN || c->bytes > SIZE_MAX - cr_aligned(size))) { + c->failed = true; + return false; + } + return true; +} + +static bool cr_py_string(cr_ctx_t *c, const char **field) { + if (!*field) + return true; + if (c->phase == CR_COUNT && c->refs == SIZE_MAX) { + c->failed = true; + return false; + } + if (c->phase == CR_MEASURE && !cr_py_size(c, strlen(*field) + 1)) + return false; + cr_str(c, field); + return !c->failed; +} + +static void cr_walk_py_namespace(cr_ctx_t *c, CBMPyNamespaceFacts *facts) { + if (!cr_py_size(c, (size_t)facts->count * sizeof(*facts->items))) + return; + cr_array(c, (void **)&facts->items, facts->count, sizeof(*facts->items)); + if (c->failed) + return; + for (int i = 0; i < facts->count; i++) { + CBMPyNamespaceFact *f = &facts->items[i]; + if (!cr_py_string(c, &f->local_name) || !cr_py_string(c, &f->module_name) || + !cr_py_string(c, &f->member_name) || + !cr_py_size(c, (size_t)f->name_count * sizeof(*f->names))) + return; + cr_array(c, (void **)&f->names, f->name_count, sizeof(*f->names)); + if (c->failed) + return; + for (int j = 0; j < f->name_count; j++) { + if (!cr_py_string(c, &f->names[j])) + return; + } + } +} + static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition)); for (int i = 0; i < r->defs.count && r->defs.items; i++) { @@ -411,6 +459,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { } cr_str(c, &r->doc_scope); cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0); + cr_walk_py_namespace(c, &r->py_namespace); } void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) { @@ -493,7 +542,8 @@ static void cr_scratch_put(cr_ctx_t *c) { } void cbm_result_compact(CBMFileResult *result) { - if (!result || result->arena.nblocks == 0) { + if (!result || result->arena.nblocks == 0 || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { return; } cr_ctx_t c; @@ -505,14 +555,23 @@ void cbm_result_compact(CBMFileResult *result) { c.phase = CR_COUNT; cr_walk(&c, &tmp); + /* Bound intern-table/replay sizing before the existing power-of-two step. */ + if (c.failed || c.refs > (SIZE_MAX / (sizeof(cr_slot_t) * 8) - CR_MIN_TABLE)) + return; c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE); + if (c.cap > UINT32_MAX) /* replay entries store slot indexes in uint32_t */ + return; if (!cr_scratch_get(&c)) { return; } c.phase = CR_MEASURE; cr_walk(&c, &tmp); + if (c.failed) { + cr_scratch_put(&c); + return; + } for (size_t i = 0; i < c.cap; i++) { c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */ } @@ -537,6 +596,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.defs.cap = tmp.defs.count; tmp.calls.cap = tmp.calls.count; tmp.imports.cap = tmp.imports.count; + tmp.py_namespace.cap = tmp.py_namespace.count; tmp.usages.cap = tmp.usages.count; tmp.throws.cap = tmp.throws.count; tmp.rw.cap = tmp.rw.count; diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c index 33a732451..503e1fb2a 100644 --- a/internal/cbm/result_spill.c +++ b/internal/cbm/result_spill.c @@ -258,7 +258,8 @@ bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFile * result that owns sub-results (embedded languages) has more arenas than * that and stays in memory. A retained parse tree is a re-parse cache, * not data: it is dropped with the in-memory result below. */ - if (result->arena.nblocks != 1 || result->owned_result_count != 0) { + if (result->arena.nblocks != 1 || result->owned_result_count != 0 || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { return false; } /* The disk ceiling. Refusing a park leaves the result in memory, which is @@ -356,6 +357,10 @@ CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) { } cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len, r->arena.blocks[0]); + if (!cbm_py_namespace_facts_valid(&r->py_namespace)) { + cbm_free_result(r); + return NULL; + } atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed); return r; } diff --git a/internal/cbm/result_spill.h b/internal/cbm/result_spill.h index 88cfd3720..8777a5bca 100644 --- a/internal/cbm/result_spill.h +++ b/internal/cbm/result_spill.h @@ -43,7 +43,10 @@ CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot); /* Read only the parked header of slot `slot` into *out: every count in it * is valid, every pointer meaningless. False when the slot is empty or the - * read fails. Lets a consumer skip the load when there is nothing to read. */ + * read fails. Lets a consumer skip the load when there is nothing to read. + * Python namespace version/language/status/count are scalar evidence only: + * zero facts with NOT_CAPTURED/INCOMPLETE is not a known-empty namespace, and + * event pointers require load. Failed peek/load provides no namespace evidence. */ bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out); /* Read only the parked header of slot `slot`: how many defs and impl diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c index c96d48ad9..67bffd5f7 100644 --- a/src/pipeline/lsp_surface.c +++ b/src/pipeline/lsp_surface.c @@ -19,6 +19,7 @@ #include "pipeline/lsp_surface.h" #include "pipeline/pipeline_internal.h" +#include #include #include #include @@ -32,7 +33,7 @@ #include "pipeline/worker_pool.h" #include "yyjson/yyjson.h" -enum { SURFACE_CODEC_VERSION = 1 }; +enum { SURFACE_CODEC_VERSION = 2 }; /* Labels the incremental name registry serves that pxc_map_label does NOT * carry into the CBMLSPDef set. Their (name, qn, label) triple must still @@ -74,10 +75,288 @@ static void add_str_array_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, cons yyjson_mut_obj_add_val(doc, obj, key, arr); } +/* The namespace payload is content-only. Its ordered records never depend on + * the registry used to build ordinary LSP definitions. */ +static bool surface_py_add_nullable(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, + const char *value) { + return value ? yyjson_mut_obj_add_str(doc, obj, key, value) + : yyjson_mut_obj_add_null(doc, obj, key); +} + +static bool surface_add_python_namespace(yyjson_mut_doc *doc, yyjson_mut_val *root, + const CBMFileResult *result, CBMLanguage language, + const char *rel_path) { + if (language != CBM_LANG_PYTHON) { + return yyjson_mut_obj_add_null(doc, root, "py"); + } + if (!result || !result->module_qn || !result->module_qn[0] || !rel_path || !rel_path[0] || + result->py_namespace.status != CBM_PY_NS_COMPLETE || + result->py_namespace.language != CBM_LANG_PYTHON || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { + return false; + } + const CBMPyNamespaceFacts *facts = &result->py_namespace; + yyjson_mut_val *py = yyjson_mut_obj(doc); + yyjson_mut_val *events = yyjson_mut_arr(doc); + if (!py || !events || !yyjson_mut_obj_add_uint(doc, py, "v", facts->version) || + !yyjson_mut_obj_add_int(doc, py, "lang", (int)facts->language) || + !yyjson_mut_obj_add_str(doc, py, "module", result->module_qn) || + !yyjson_mut_obj_add_str(doc, py, "path", rel_path) || + !yyjson_mut_obj_add_int(doc, py, "status", (int)facts->status) || + !yyjson_mut_obj_add_int(doc, py, "failure", (int)facts->failure)) { + return false; + } + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + yyjson_mut_val *event = yyjson_mut_obj(doc); + yyjson_mut_val *names = yyjson_mut_arr(doc); + if (!event || !names || !yyjson_mut_obj_add_int(doc, event, "k", (int)f->kind) || + !yyjson_mut_obj_add_int(doc, event, "d", (int)f->def_kind) || + !yyjson_mut_obj_add_int(doc, event, "s", (int)f->sequence_kind) || + !yyjson_mut_obj_add_int(doc, event, "r", (int)f->reason) || + !yyjson_mut_obj_add_uint(doc, event, "f", f->flags) || + !yyjson_mut_obj_add_uint(doc, event, "l", f->relative_level) || + !surface_py_add_nullable(doc, event, "n", f->local_name) || + !surface_py_add_nullable(doc, event, "m", f->module_name) || + !surface_py_add_nullable(doc, event, "i", f->member_name)) { + return false; + } + for (int j = 0; j < f->name_count; j++) { + if (!yyjson_mut_arr_add_str(doc, names, f->names[j])) { + return false; + } + } + if (!yyjson_mut_obj_add_val(doc, event, "a", names) || + !yyjson_mut_arr_add_val(events, event)) { + return false; + } + } + return yyjson_mut_obj_add_val(doc, py, "events", events) && + yyjson_mut_obj_add_val(doc, root, "py", py); +} + +/* Reject duplicate fields rather than letting object lookup choose one. */ +static yyjson_val *surface_unique_field(yyjson_val *obj, const char *name) { + if (!yyjson_is_obj(obj)) { + return NULL; + } + yyjson_val *found = NULL; + yyjson_obj_iter it = yyjson_obj_iter_with(obj); + yyjson_val *key; + while ((key = yyjson_obj_iter_next(&it))) { + const char *text = yyjson_get_str(key); + if (text && yyjson_get_len(key) == strlen(name) && strcmp(text, name) == 0) { + if (found) { + return NULL; + } + found = yyjson_obj_iter_get_val(key); + } + } + return found; +} + +static bool surface_py_uint(yyjson_val *value, uint32_t maximum, uint32_t *out) { + if (!yyjson_is_int(value)) { + return false; + } + int64_t n = yyjson_get_sint(value); + if (n < 0 || (uint64_t)n > maximum) { + return false; + } + *out = (uint32_t)n; + return true; +} + +static bool surface_py_string(yyjson_val *value, bool nullable, const char **out) { + *out = NULL; + if (!value) { + return false; + } + if (yyjson_is_null(value)) { + return nullable; + } + const char *text = yyjson_get_str(value); + if (!text || strlen(text) != yyjson_get_len(value)) { + return false; + } + *out = text; + return true; +} + +/* Arrays belong to scratch; strings borrow the immutable JSON document. + * The caller keeps both alive until validation/copy/comparison is finished. */ +static int surface_python_namespace_parse(CBMArena *scratch, yyjson_val *root, + cbm_lsp_python_namespace_t *out) { + memset(out, 0, sizeof(*out)); + uint32_t version = 0, file_language = 0; + yyjson_val *py = surface_unique_field(root, "py"); + if (!surface_py_uint(surface_unique_field(root, "v"), UINT32_MAX, &version) || + version != SURFACE_CODEC_VERSION || !py || + !surface_py_uint(surface_unique_field(root, "lang"), CBM_LANG_COUNT - 1, &file_language) || + !yyjson_is_arr(surface_unique_field(root, "lsp"))) { + return -1; + } + if (yyjson_is_null(py)) { + return file_language == CBM_LANG_PYTHON ? -1 : 0; + } + if (file_language != CBM_LANG_PYTHON || !yyjson_is_obj(py) || yyjson_obj_size(py) != 7) { + return -1; + } + CBMPyNamespaceFacts *facts = &out->facts; + uint32_t language, status, failure; + if (!surface_py_uint(surface_unique_field(py, "v"), UINT32_MAX, &facts->version) || + !surface_py_uint(surface_unique_field(py, "lang"), INT_MAX, &language) || + !surface_py_uint(surface_unique_field(py, "status"), INT_MAX, &status) || + !surface_py_uint(surface_unique_field(py, "failure"), INT_MAX, &failure) || + !surface_py_string(surface_unique_field(py, "module"), false, &out->module_qn) || + !surface_py_string(surface_unique_field(py, "path"), false, &out->rel_path) || + !out->module_qn[0] || !out->rel_path[0] || language != CBM_LANG_PYTHON || + status != CBM_PY_NS_COMPLETE || failure != CBM_PY_NS_FAILURE_NONE) { + return -1; + } + facts->language = (CBMLanguage)language; + facts->status = (CBMPyNamespaceStatus)status; + facts->failure = (CBMPyNamespaceFailure)failure; + yyjson_val *events = surface_unique_field(py, "events"); + if (!yyjson_is_arr(events)) { + return -1; + } + size_t count = yyjson_arr_size(events); + if (count > INT_MAX || count > SIZE_MAX / sizeof(*facts->items)) { + return -1; + } + facts->count = (int)count; + facts->cap = (int)count; + if (count) { + facts->items = cbm_arena_alloc(scratch, count * sizeof(*facts->items)); + if (!facts->items) { + return -1; + } + memset(facts->items, 0, count * sizeof(*facts->items)); + } + yyjson_arr_iter it = yyjson_arr_iter_with(events); + yyjson_val *event; + int i = 0; + while ((event = yyjson_arr_iter_next(&it))) { + if (!yyjson_is_obj(event) || yyjson_obj_size(event) != 10) { + return -1; + } + CBMPyNamespaceFact *f = &facts->items[i++]; + uint32_t kind, def_kind, sequence_kind, reason; + if (!surface_py_uint(surface_unique_field(event, "k"), INT_MAX, &kind) || + !surface_py_uint(surface_unique_field(event, "d"), INT_MAX, &def_kind) || + !surface_py_uint(surface_unique_field(event, "s"), INT_MAX, &sequence_kind) || + !surface_py_uint(surface_unique_field(event, "r"), INT_MAX, &reason) || + !surface_py_uint(surface_unique_field(event, "f"), UINT32_MAX, &f->flags) || + !surface_py_uint(surface_unique_field(event, "l"), UINT32_MAX, &f->relative_level) || + !surface_py_string(surface_unique_field(event, "n"), true, &f->local_name) || + !surface_py_string(surface_unique_field(event, "m"), true, &f->module_name) || + !surface_py_string(surface_unique_field(event, "i"), true, &f->member_name)) { + return -1; + } + f->kind = (CBMPyNamespaceFactKind)kind; + f->def_kind = (CBMPyNamespaceDefKind)def_kind; + f->sequence_kind = (CBMPyNamespaceSequenceKind)sequence_kind; + f->reason = (CBMPyNamespaceUnknownReason)reason; + yyjson_val *names = surface_unique_field(event, "a"); + if (!yyjson_is_arr(names)) { + return -1; + } + size_t n = yyjson_arr_size(names); + if (n > INT_MAX || n > SIZE_MAX / sizeof(*f->names)) { + return -1; + } + f->name_count = (int)n; + if (n) { + f->names = cbm_arena_alloc(scratch, n * sizeof(*f->names)); + if (!f->names) { + return -1; + } + } + yyjson_arr_iter ni = yyjson_arr_iter_with(names); + yyjson_val *name; + int j = 0; + while ((name = yyjson_arr_iter_next(&ni))) { + if (!surface_py_string(name, false, &f->names[j++])) { + return -1; + } + } + } + return cbm_py_namespace_facts_valid(facts) ? 1 : -1; +} + +int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json, + cbm_lsp_python_namespace_t *out) { + if (!out) { + return -1; + } + memset(out, 0, sizeof(*out)); + if (!arena || !json) { + return -1; + } + yyjson_doc *doc = yyjson_read(json, strlen(json), 0); + if (!doc) { + return -1; + } + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_lsp_python_namespace_t parsed = {0}, result = {0}; + int status = surface_python_namespace_parse(&scratch, yyjson_doc_get_root(doc), &parsed); + if (status == 1) { + if (!cbm_py_namespace_facts_copy(arena, &parsed.facts, &result.facts)) { + status = -1; + } else { + result.module_qn = cbm_arena_strdup(arena, parsed.module_qn); + result.rel_path = cbm_arena_strdup(arena, parsed.rel_path); + if (!result.module_qn || !result.rel_path) { + status = -1; + } + } + } + cbm_arena_destroy(&scratch); + yyjson_doc_free(doc); + if (status == 1) { + *out = result; + } + return status; +} + +int cbm_lsp_surface_python_change(const char *old_json, const char *new_json) { + if (!old_json) { + return -1; + } + yyjson_doc *old_doc = yyjson_read(old_json, strlen(old_json), 0); + yyjson_doc *new_doc = new_json ? yyjson_read(new_json, strlen(new_json), 0) : NULL; + if (!old_doc || (new_json && !new_doc)) { + yyjson_doc_free(old_doc); + yyjson_doc_free(new_doc); + return -1; + } + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_lsp_python_namespace_t old_facts, new_facts; + int old_state = + surface_python_namespace_parse(&scratch, yyjson_doc_get_root(old_doc), &old_facts); + int new_state = + new_doc ? surface_python_namespace_parse(&scratch, yyjson_doc_get_root(new_doc), &new_facts) + : 0; + int result = -1; + if (old_state >= 0 && new_state >= 0) { + result = (old_state == 0 && new_state == 0) ? 0 + : (old_state != new_state || !new_json || strcmp(old_json, new_json) != 0) ? 1 + : 0; + } + cbm_arena_destroy(&scratch); + yyjson_doc_free(old_doc); + yyjson_doc_free(new_doc); + return result; +} + /* Serialize one file's surface: its slice of all_defs plus the registry-only * symbols from its raw extraction defs. Returns a malloc'd JSON string and its * length. */ -static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef *defs, int def_count, +static char *surface_file_to_json(const CBMFileResult *result, CBMLanguage language, + const char *rel_path, const CBMLSPDef *defs, int def_count, size_t *out_len) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); if (!doc) { @@ -85,7 +364,12 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * } yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION); + if ((unsigned)language >= CBM_LANG_COUNT || + !yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION) || + !yyjson_mut_obj_add_int(doc, root, "lang", (int)language)) { + yyjson_mut_doc_free(doc); + return NULL; + } yyjson_mut_val *lsp = yyjson_mut_arr(doc); for (int i = 0; i < def_count; i++) { @@ -177,6 +461,10 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * cbm_free(CBM_MEM_CLASS_OTHER, portable); } + if (!surface_add_python_namespace(doc, root, result, language, rel_path)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *json = yyjson_mut_write(doc, 0, out_len); yyjson_mut_doc_free(doc); return json; @@ -213,8 +501,9 @@ static void surface_row_one(int i, void *arg) { int start = job->def_starts ? job->def_starts[i] : 0; int end = job->def_starts ? job->def_starts[i + 1] : 0; size_t json_len = 0; - char *json = surface_file_to_json(fr, job->all_defs ? job->all_defs + start : NULL, end - start, - &json_len); + char *json = + surface_file_to_json(fr, job->files[i].language, job->files[i].rel_path, + job->all_defs ? job->all_defs + start : NULL, end - start, &json_len); cbm_pipeline_result_release(fr, loaded); if (!json) { atomic_store_explicit(&job->failed, true, memory_order_relaxed); @@ -329,9 +618,14 @@ int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLS return -1; } yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *ver = root ? yyjson_obj_get(root, "v") : NULL; - yyjson_val *lsp = root ? yyjson_obj_get(root, "lsp") : NULL; - if (!ver || yyjson_get_int(ver) != SURFACE_CODEC_VERSION || !lsp || !yyjson_is_arr(lsp)) { + CBMArena namespace_scratch; + cbm_arena_init(&namespace_scratch); + cbm_lsp_python_namespace_t namespace_payload = {0}; + int namespace_status = + surface_python_namespace_parse(&namespace_scratch, root, &namespace_payload); + cbm_arena_destroy(&namespace_scratch); + yyjson_val *lsp = surface_unique_field(root, "lsp"); + if (namespace_status < 0) { yyjson_doc_free(doc); return -1; } diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.h index de3390d85..dc198bb65 100644 --- a/src/pipeline/lsp_surface.h +++ b/src/pipeline/lsp_surface.h @@ -10,8 +10,10 @@ * unchanged — a body edit — and skip recomputing its dependents, and * (b) rehydrate the cross registries for files it does not re-parse. * - * The JSON is a versioned object {"v":1,"lsp":[...],"reg":[...]}. Field - * order and array order are fixed by the writer, so byte equality of the + * The JSON is a versioned object {"v":2,"lang":N,"lsp":[...],"reg":[...],"py":...}. + * Python rows carry content-only namespace facts, including ordered binding + * events and unknown effects; other languages carry an explicit null. + * Field order and array order are fixed by the writer, so byte equality of the * serialization is surface equality; the sha256 of these bytes is the * stored surface_sha. Rows written by a different codec version fail the * "v" check on load and route the run to a full rebuild. @@ -28,7 +30,8 @@ * all_defs[def_starts[i] .. def_starts[i+1]) are file i's defs. Rows carry * heap strings; release with cbm_store_free_lsp_surfaces. Files with an * empty surface still get a row (empty arrays hash too — "no defs" must be - * distinguishable from "no data"). Returns 0, or -1 on allocation failure. */ + * distinguishable from "no data"). Returns 0, or -1 on allocation failure + * or invalid/incomplete Python namespace metadata. */ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project, CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const CBMLSPDef *all_defs, const int *def_starts, @@ -40,4 +43,29 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec * different codec version — callers route that to a full rebuild. */ int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs); +/* A restored Python namespace retains its raw module identity and source path. + * Facts and origin strings live in the caller arena; no JSON storage is borrowed. */ +typedef struct { + const char *module_qn; + const char *rel_path; + CBMPyNamespaceFacts facts; +} cbm_lsp_python_namespace_t; + +/* Returns 1 for a complete Python payload, 0 for a valid non-Python row, or + * -1 for invalid/incomplete metadata or allocation failure. Output is empty + * unless a complete Python payload was copied successfully. */ +int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json, + cbm_lsp_python_namespace_t *out); + +/* Conservative namespace admission for incremental planning. Both rows must + * have a current, valid fact envelope before any equality shortcut is used. + * Returns -1 for invalid/incomplete metadata, 0 when both rows are non-Python + * or their complete Python surfaces are identical, and 1 for a Python surface + * change or a language transition involving Python. A NULL new_json denotes + * deletion: valid Python metadata returns 1, valid non-Python metadata 0. + * NULL old_json is invalid. The full Python surface comparison deliberately + * includes definition properties until a narrower dependency closure is proven. + */ +int cbm_lsp_surface_python_change(const char *old_json, const char *new_json); + #endif /* CBM_PIPELINE_LSP_SURFACE_H */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index a04cc7044..30a5a6620 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2694,8 +2694,9 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p goto done; } - /* Early cutoff per changed file; surface-changed files seed the - * dependent query. Deleted files always seed it. */ + /* Validate Python namespace metadata before any hash cutoff or dependent + * query. Python changes require a full rebuild; other surface changes and + * deletions seed the dependent query. */ int n_surface_changed = 0; /* dep_targets, then the body-edited files (body_targets) in the same block */ const char **dep_targets = @@ -2722,6 +2723,15 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p decline = "missing_surface_row"; goto done; } + int python_change = + cbm_lsp_surface_python_change(stored_row->defs_json, fresh_row->defs_json); + if (python_change != 0) { + free(dep_targets); + cbm_ht_free(rows_by_path); + decline = python_change < 0 ? "python_namespace_metadata_invalid" + : "python_namespace_changed"; + goto done; + } if (strcmp(stored_row->surface_sha, fresh_row->surface_sha) == 0) { /* body edit: the file re-resolves, and of the others only the * documents bound to its lines (below) */ @@ -2748,6 +2758,17 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p } bool gone_scope_changed = false; /* a deleted file takes its declarations along */ for (int i = 0; i < n_deleted; i++) { + const cbm_lsp_surface_row_t *stored_row = + cbm_ht_get(rows_by_path, changed_paths[n_changed + i]); + int python_change = + cbm_lsp_surface_python_change(stored_row ? stored_row->defs_json : NULL, NULL); + if (python_change != 0) { + free(dep_targets); + cbm_ht_free(rows_by_path); + decline = python_change < 0 ? "python_namespace_metadata_invalid" + : "python_namespace_changed"; + goto done; + } dep_targets[dep_target_count++] = changed_paths[n_changed + i]; const cbm_lsp_surface_row_t *gone = cbm_ht_get(rows_by_path, changed_paths[n_changed + i]); if (gone && !doc_scope_repairable(gone->defs_json, NULL, removed_names)) { diff --git a/tests/test_extraction.c b/tests/test_extraction.c index aa53b5c87..95bd26220 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -10987,6 +10987,485 @@ TEST(extract_enclosing_sweep_matches_scan_issue1527) { PASS(); } +/* Python namespace facts: provenance is ordered, content-only and independently + * owned. These tests do not enable any reexport resolution. */ +static bool py_facts_equal(const CBMPyNamespaceFacts *a, const CBMPyNamespaceFacts *b) { + if (!cbm_py_namespace_facts_valid(a) || !cbm_py_namespace_facts_valid(b) || + a->version != b->version || a->language != b->language || a->status != b->status || + a->failure != b->failure || a->count != b->count) + return false; + for (int i = 0; i < a->count; i++) { + const CBMPyNamespaceFact *x = &a->items[i], *y = &b->items[i]; + if (x->kind != y->kind || x->def_kind != y->def_kind || + x->sequence_kind != y->sequence_kind || x->reason != y->reason || + x->flags != y->flags || x->relative_level != y->relative_level || + x->name_count != y->name_count || !cmp_str_eq(x->local_name, y->local_name) || + !cmp_str_eq(x->module_name, y->module_name) || + !cmp_str_eq(x->member_name, y->member_name)) + return false; + for (int j = 0; j < x->name_count; j++) { + if (!cmp_str_eq(x->names[j], y->names[j])) + return false; + } + } + return true; +} + +static bool py_facts_owned_range(const CBMArena *arena, const void *ptr, size_t bytes) { + if (!ptr) + return bytes == 0; + uintptr_t address = (uintptr_t)ptr; + for (int i = 0; i < arena->nblocks; i++) { + uintptr_t start = (uintptr_t)arena->blocks[i]; + size_t size = arena->block_sizes[i]; + if (address >= start && bytes <= size && address - start <= size - bytes) + return true; + } + return false; +} + +static bool py_facts_owned_string(const CBMArena *arena, const char *str) { + return !str || (py_facts_owned_range(arena, str, 1) && + py_facts_owned_range(arena, str, strlen(str) + 1)); +} + +static bool py_facts_owned(const CBMArena *arena, const CBMPyNamespaceFacts *facts) { + if (!py_facts_owned_range(arena, facts->items, (size_t)facts->count * sizeof(*facts->items))) + return false; + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + if (!py_facts_owned_string(arena, f->local_name) || + !py_facts_owned_string(arena, f->module_name) || + !py_facts_owned_string(arena, f->member_name) || + !py_facts_owned_range(arena, f->names, (size_t)f->name_count * sizeof(*f->names))) + return false; + for (int j = 0; j < f->name_count; j++) { + if (!py_facts_owned_string(arena, f->names[j])) + return false; + } + } + return true; +} + +static const char PY_FACTS_ORDERED[] = "def item():\n return 1\n" + "from first import item\n" + "from second import item as item\n" + "item = unknown\n" + "del item\n" + "import pkg.mod\n" + "import pkg.mod as alias\n" + "from . import child as local\n" + "from ..outer import member\n" + "from one.shared import *\n" + "from two.shared import *\n" + "class Local:\n pass\n"; + +TEST(python_namespace_facts_preserve_order_and_import_provenance) { + CBMFileResult *r = extract(PY_FACTS_ORDERED, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + bool ok = r && !r->has_error && !r->parse_incomplete && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 12 && + cbm_py_namespace_facts_valid(&r->py_namespace) && has_def(r, "Function", "item") && + has_def(r, "Class", "Local") && r->module_qn && + strcmp(r->module_qn, "t.pkg.__init__") == 0; + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_OWN_DEF && f[0].def_kind == CBM_PY_NS_DEF_FUNCTION && + strcmp(f[0].local_name, "item") == 0 && f[1].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[1].module_name, "first") == 0 && strcmp(f[1].member_name, "item") == 0 && + strcmp(f[1].local_name, "item") == 0 && f[2].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[2].module_name, "second") == 0 && strcmp(f[2].local_name, "item") == 0 && + f[3].kind == CBM_PY_NS_SHADOW && strcmp(f[3].local_name, "item") == 0 && + f[4].kind == CBM_PY_NS_DELETE && strcmp(f[4].local_name, "item") == 0 && + f[5].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[5].module_name, "pkg.mod") == 0 && + strcmp(f[5].local_name, "pkg") == 0 && f[5].flags == CBM_PY_NS_IMPORT_BINDS_ROOT && + f[6].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[6].module_name, "pkg.mod") == 0 && + strcmp(f[6].local_name, "alias") == 0 && f[6].flags == 0 && f[7].relative_level == 1 && + strcmp(f[7].module_name, "") == 0 && strcmp(f[7].member_name, "child") == 0 && + strcmp(f[7].local_name, "local") == 0 && f[8].relative_level == 2 && + strcmp(f[8].module_name, "outer") == 0 && strcmp(f[8].member_name, "member") == 0 && + f[9].kind == CBM_PY_NS_IMPORT_STAR && strcmp(f[9].module_name, "one.shared") == 0 && + f[10].kind == CBM_PY_NS_IMPORT_STAR && strcmp(f[10].module_name, "two.shared") == 0 && + f[11].kind == CBM_PY_NS_OWN_DEF && f[11].def_kind == CBM_PY_NS_DEF_CLASS && + strcmp(f[11].local_name, "Local") == 0 && py_facts_owned(&r->arena, &r->py_namespace); + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_distinguish_all_operations_and_unknowns) { + static const char source[] = "__all__ = ['old']; __all__ = []\n" + "__all__ = dynamic\n" + "__all__ = ('new', '_private', 'a,b', '')\n" + "__all__ += ('new',)\n" + "__all__ -= []\n" + "del __all__\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + CBMFileResult *empty = extract("# no bindings\n", CBM_LANG_PYTHON, "t", "empty.py"); + bool ok = r && empty && !r->has_error && !r->parse_incomplete && !empty->has_error && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 7 && + empty->py_namespace.status == CBM_PY_NS_COMPLETE && empty->py_namespace.count == 0 && + cbm_py_namespace_facts_valid(&r->py_namespace); + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_ALL_SET && f[0].name_count == 1 && + strcmp(f[0].names[0], "old") == 0 && f[1].kind == CBM_PY_NS_ALL_SET && + f[1].name_count == 0 && f[1].names == NULL && + f[1].sequence_kind == CBM_PY_NS_SEQUENCE_LIST && f[2].kind == CBM_PY_NS_ALL_UNKNOWN && + f[3].kind == CBM_PY_NS_ALL_SET && f[3].sequence_kind == CBM_PY_NS_SEQUENCE_TUPLE && + f[3].name_count == 4 && strcmp(f[3].names[0], "new") == 0 && + strcmp(f[3].names[1], "_private") == 0 && strcmp(f[3].names[2], "a,b") == 0 && + strcmp(f[3].names[3], "") == 0 && f[4].kind == CBM_PY_NS_ALL_APPEND && + f[4].name_count == 1 && strcmp(f[4].names[0], "new") == 0 && + f[5].kind == CBM_PY_NS_ALL_UNKNOWN && f[6].kind == CBM_PY_NS_ALL_DELETE; + } + if (r) + cbm_free_result(r); + if (empty) + cbm_free_result(empty); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_unknown_effects_and_body_stability) { + static const char one[] = "from pkg import Item\ndef f():\n return 1\n"; + static const char two[] = + "from pkg import Item\ndef f():\n from elsewhere import Other\n return Other()\n"; + CBMFileResult *a = extract(one, CBM_LANG_PYTHON, "t", "main.py"); + CBMFileResult *b = extract(two, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = a && b && !a->has_error && !b->has_error && !a->parse_incomplete && + !b->parse_incomplete && a->py_namespace.count == 2 && + py_facts_equal(&a->py_namespace, &b->py_namespace); + if (a) + cbm_free_result(a); + if (b) + cbm_free_result(b); + static const char unknown[] = "from pkg import Item\n" + "if enabled:\n from other import Item\n" + "@deco\ndef wrapped():\n pass\n" + "class Derived(Base):\n pass\n" + "Item = factory()\n"; + a = extract(unknown, CBM_LANG_PYTHON, "t", "main.py"); + ok = a && !a->has_error && !a->parse_incomplete && a->py_namespace.count == 8 && ok; + if (a && a->py_namespace.count == 8) { + const CBMPyNamespaceFact *f = a->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_IMPORT_NAME && f[1].kind == CBM_PY_NS_UNKNOWN && + f[1].reason == CBM_PY_NS_REASON_COMPOUND && f[2].kind == CBM_PY_NS_UNKNOWN && + f[3].kind == CBM_PY_NS_SHADOW && f[3].reason == CBM_PY_NS_REASON_DECORATED && + strcmp(f[3].local_name, "wrapped") == 0 && f[4].kind == CBM_PY_NS_UNKNOWN && + f[5].kind == CBM_PY_NS_SHADOW && f[5].reason == CBM_PY_NS_REASON_UNPROVEN_CLASS && + f[6].kind == CBM_PY_NS_UNKNOWN && f[7].kind == CBM_PY_NS_SHADOW && ok; + } + if (a) + cbm_free_result(a); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_parse_errors_are_complete_unknown) { + const char *source = "from pkg import Before\ndef broken(:\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = r && r->parse_incomplete && r->py_namespace.status == CBM_PY_NS_COMPLETE && + r->py_namespace.count == 1 && r->py_namespace.items[0].kind == CBM_PY_NS_UNKNOWN && + r->py_namespace.items[0].reason == CBM_PY_NS_REASON_PARSE && + cbm_py_namespace_facts_valid(&r->py_namespace); + if (r) + cbm_free_result(r); + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + CBMExtractCtx ctx = {.arena = &fallback.arena, + .result = &fallback, + .source = "", + .source_len = 0, + .language = CBM_LANG_PYTHON}; + ok = cbm_extract_python_namespace_facts(&ctx) == CBM_PY_NS_COMPLETE && + fallback.py_namespace.count == 1 && + fallback.py_namespace.items[0].reason == CBM_PY_NS_REASON_PARSE && ok; + cbm_arena_destroy(&fallback.arena); + r = extract("const x = 1;\n", CBM_LANG_JAVASCRIPT, "t", "__init__.js"); + ok = r && !r->has_error && r->py_namespace.status == CBM_PY_NS_NOT_APPLICABLE && + r->py_namespace.language == CBM_LANG_JAVASCRIPT && r->py_namespace.count == 0 && ok; + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_capture_and_copy_fail_atomically) { + static const char source[] = "from .pkg import Item as Local\n__all__ = ['Local', '']\n"; + bool ok = true, capture_complete = false, copy_complete = false; + int capture_failures = 0, copy_failures = 0; + for (int limit = 0; limit < 64 && !capture_complete; limit++) { + cbm_py_namespace_test_capture_fail_after(limit); + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + cbm_py_namespace_test_capture_fail_after(-1); + if (!r) { + ok = false; + break; + } + const CBMPyNamespaceFacts *f = &r->py_namespace; + capture_complete = f->status == CBM_PY_NS_COMPLETE; + if (capture_complete) { + ok = f->count == 2 && cbm_py_namespace_facts_valid(f) && ok; + } else { + capture_failures++; + ok = f->status == CBM_PY_NS_INCOMPLETE && f->failure == CBM_PY_NS_FAILURE_ALLOCATION && + !f->items && f->count == 0 && f->cap == 0 && cbm_py_namespace_facts_valid(f) && ok; + } + cbm_free_result(r); + } + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + CBMArena reference_arena; + cbm_arena_init(&reference_arena); + CBMPyNamespaceFacts original = {0}; + ok = r && cbm_py_namespace_facts_copy(&reference_arena, &r->py_namespace, &original) && ok; + for (int limit = 0; r && limit < 64 && !copy_complete; limit++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMPyNamespaceFacts copy = {0}; + cbm_py_namespace_test_copy_fail_after(limit); + copy_complete = cbm_py_namespace_facts_copy(&arena, &r->py_namespace, ©); + cbm_py_namespace_test_copy_fail_after(-1); + if (copy_complete) { + ok = py_facts_equal(©, &r->py_namespace) && py_facts_owned(&arena, ©) && ok; + } else { + copy_failures++; + ok = copy.status == CBM_PY_NS_INCOMPLETE && + copy.failure == CBM_PY_NS_FAILURE_ALLOCATION && !copy.items && copy.count == 0 && + copy.cap == 0 && ok; + } + ok = py_facts_equal(&r->py_namespace, &original) && ok; + cbm_arena_destroy(&arena); + } + /* Rebuilding into an existing result must replace old complete facts on + * failure; do not leave a stale successful manifest attached. */ + if (r) { + CBMExtractCtx ctx = {.arena = &r->arena, + .result = r, + .source = source, + .source_len = (int)strlen(source), + .language = CBM_LANG_PYTHON}; + cbm_py_namespace_test_capture_fail_after(0); + CBMPyNamespaceStatus status = cbm_extract_python_namespace_facts(&ctx); + cbm_py_namespace_test_capture_fail_after(-1); + ok = status == CBM_PY_NS_INCOMPLETE && !r->py_namespace.items && + r->py_namespace.count == 0 && ok; + cbm_free_result(r); + } + cbm_arena_destroy(&reference_arena); + ASSERT_TRUE(ok); + ASSERT_TRUE(capture_complete); + ASSERT_TRUE(copy_complete); + ASSERT_GT(capture_failures, 2); + ASSERT_GT(copy_failures, 2); + PASS(); +} + +TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema) { + CBMPyNamespaceFacts zero = {0}; + bool ok = cbm_py_namespace_facts_valid(&zero) && zero.status == CBM_PY_NS_NOT_CAPTURED; + CBMPyNamespaceFact event = {.kind = CBM_PY_NS_ALL_SET, + .sequence_kind = CBM_PY_NS_SEQUENCE_LIST}; + CBMPyNamespaceFacts facts = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_COMPLETE, + .items = &event, + .count = 1, + .cap = 1}; + ok = cbm_py_namespace_facts_valid(&facts) && ok; + for (int variant = 0; variant < 8; variant++) { + CBMPyNamespaceFact bad_event = event; + CBMPyNamespaceFacts bad = facts; + bad.items = &bad_event; + if (variant == 0) + bad.version++; + if (variant == 1) + bad.cap = 0; + if (variant == 2) + bad.count = -1; + if (variant == 3) + bad_event.kind = (CBMPyNamespaceFactKind)999; + if (variant == 4) + bad_event.name_count = 1; + if (variant == 5) + bad_event.flags = UINT32_MAX; + if (variant == 6) + bad.language = CBM_LANG_JAVA; + if (variant == 7) + bad.status = CBM_PY_NS_INCOMPLETE; + CBMArena arena; + cbm_arena_init(&arena); + CBMPyNamespaceFacts copy = {0}; + ok = !cbm_py_namespace_facts_valid(&bad) && + !cbm_py_namespace_facts_copy(&arena, &bad, ©) && + copy.status == CBM_PY_NS_INCOMPLETE && copy.count == 0 && !copy.items && ok; + cbm_arena_destroy(&arena); + } + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload) { + char source[] = "from . import Thing as Local\n" + "__all__ = ['Local', '', 'a,b', '_private']\n" + "if enabled:\n Local = dynamic\n" + "__all__ = []\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + CBMArena retained; + cbm_arena_init(&retained); + CBMPyNamespaceFacts expected = {0}; + bool ok = r && !r->has_error && !r->parse_incomplete && r->py_namespace.count == 4 && + cbm_py_namespace_facts_copy(&retained, &r->py_namespace, &expected); + memset(source, 'x', strlen(source)); /* facts cannot borrow the caller's input */ + char dir[512]; + int n = snprintf(dir, sizeof(dir), "%s/cbm_py_facts_XXXXXX", cbm_tmpdir()); + bool made = n > 0 && (size_t)n < sizeof(dir) && cbm_mkdtemp(dir); + size_t saved = cbm_result_spill_pin_free_bytes_for_tests(SIZE_MAX); + cbm_result_spill_t *spill = made ? cbm_result_spill_open(dir, 1, 1) : NULL; + cbm_result_spill_pin_free_bytes_for_tests(saved); + ok = spill && ok; + if (r && spill && ok) { + cbm_result_compact(r); + cbm_result_compact(r); + ok = r->arena.nblocks == 1 && r->py_namespace.cap == r->py_namespace.count && + py_facts_equal(&r->py_namespace, &expected) && + py_facts_owned(&r->arena, &r->py_namespace); + if (ok && cbm_result_spill_park(spill, 0, 0, r)) { + r = NULL; + CBMFileResult header = {0}; + ok = cbm_result_spill_peek_header(spill, 0, &header) && + header.py_namespace.status == CBM_PY_NS_COMPLETE && header.py_namespace.count == 4; + for (int round = 0; round < 2; round++) { + CBMFileResult *loaded = cbm_result_spill_load(spill, 0); + ok = loaded && loaded->cached_tree == NULL && + py_facts_owned(&loaded->arena, &loaded->py_namespace) && + py_facts_equal(&loaded->py_namespace, &expected) && ok; + if (loaded) + cbm_free_result(loaded); + } + } else { + ok = false; + } + } + if (r) + cbm_free_result(r); + cbm_result_spill_close(spill); + if (made) { + char subdir[1024]; + snprintf(subdir, sizeof(subdir), "%s/spill", dir); + cbm_rmdir(subdir); + cbm_rmdir(dir); + } + /* Original result/source tree and both reloads are gone; the independent + * copied payload still owns the relative empty module and literal strings. */ + ok = py_facts_owned(&retained, &expected) && cbm_py_namespace_facts_valid(&expected) && + expected.count == 4 && expected.items[0].relative_level == 1 && + strcmp(expected.items[0].module_name, "") == 0 && expected.items[1].name_count == 4 && + strcmp(expected.items[1].names[2], "a,b") == 0 && ok; + cbm_arena_destroy(&retained); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_unavailable_and_empty_survive_spill) { + CBMPyNamespaceFacts states[] = { + {0}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_COMPLETE}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_INCOMPLETE, + .failure = CBM_PY_NS_FAILURE_ALLOCATION}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_JAVASCRIPT, + .status = CBM_PY_NS_NOT_APPLICABLE}, + }; + char dir[512]; + int n = snprintf(dir, sizeof(dir), "%s/cbm_py_fact_states_XXXXXX", cbm_tmpdir()); + bool made = n > 0 && (size_t)n < sizeof(dir) && cbm_mkdtemp(dir); + size_t saved = cbm_result_spill_pin_free_bytes_for_tests(SIZE_MAX); + cbm_result_spill_t *spill = made ? cbm_result_spill_open(dir, 1, 4) : NULL; + cbm_result_spill_pin_free_bytes_for_tests(saved); + bool ok = spill != NULL; + for (int i = 0; spill && i < 4; i++) { + CBMFileResult *r = cbm_result_alloc(); + if (!r) { + ok = false; + break; + } + cbm_arena_init(&r->arena); + r->module_qn = cbm_arena_strdup(&r->arena, "t.metadata"); + r->py_namespace = states[i]; + cbm_result_compact(r); + cbm_result_compact(r); + bool ready = + r->module_qn && r->arena.nblocks == 1 && py_facts_equal(&r->py_namespace, &states[i]); + if (!ready || !cbm_result_spill_park(spill, 0, i, r)) { + ok = false; + cbm_free_result(r); + continue; + } + CBMFileResult header = {0}; + ok = cbm_result_spill_peek_header(spill, i, &header) && + header.py_namespace.status == states[i].status && header.py_namespace.count == 0 && ok; + for (int round = 0; round < 2; round++) { + CBMFileResult *loaded = cbm_result_spill_load(spill, i); + ok = loaded && py_facts_equal(&loaded->py_namespace, &states[i]) && + py_facts_owned(&loaded->arena, &loaded->py_namespace) && ok; + if (loaded) + cbm_free_result(loaded); + } + } + cbm_result_spill_close(spill); + if (made) { + char subdir[1024]; + snprintf(subdir, sizeof(subdir), "%s/spill", dir); + cbm_rmdir(subdir); + cbm_rmdir(dir); + } + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_multiple_bindings_and_unsupported_effects) { + static const char source[] = "import alpha, beta.mod as Beta\n" + "from pkg import (One, Two as Local)\n" + "from pkg import later\n" + "def later():\n pass\n" + "from pkg import __all__\n" + "def __all__():\n pass\n" + "values = {obj: 1}\n" + "class HasDescriptor:\n field = descriptor\n" + "__all__ = ['escaped\\n']\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = r && !r->has_error && !r->parse_incomplete && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 14 && + cbm_py_namespace_facts_valid(&r->py_namespace) && has_def(r, "Function", "later") && + has_def(r, "Class", "HasDescriptor"); + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[0].local_name, "alpha") == 0 && + f[1].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[1].local_name, "Beta") == 0 && + strcmp(f[1].module_name, "beta.mod") == 0 && f[2].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[2].member_name, "One") == 0 && f[3].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[3].member_name, "Two") == 0 && strcmp(f[3].local_name, "Local") == 0 && + f[4].kind == CBM_PY_NS_IMPORT_NAME && f[5].kind == CBM_PY_NS_OWN_DEF && + strcmp(f[5].local_name, "later") == 0 && f[6].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[6].local_name, "__all__") == 0 && f[7].kind == CBM_PY_NS_OWN_DEF && + strcmp(f[7].local_name, "__all__") == 0 && f[8].kind == CBM_PY_NS_UNKNOWN && + f[9].kind == CBM_PY_NS_SHADOW && strcmp(f[9].local_name, "values") == 0 && + f[10].kind == CBM_PY_NS_UNKNOWN && f[11].kind == CBM_PY_NS_SHADOW && + f[11].reason == CBM_PY_NS_REASON_UNPROVEN_CLASS && f[12].kind == CBM_PY_NS_UNKNOWN && + f[13].kind == CBM_PY_NS_ALL_UNKNOWN; + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + SUITE(extraction) { RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); @@ -10997,6 +11476,16 @@ SUITE(extraction) { /* Initialize extraction library */ cbm_init(); + RUN_TEST(python_namespace_facts_preserve_order_and_import_provenance); + RUN_TEST(python_namespace_facts_distinguish_all_operations_and_unknowns); + RUN_TEST(python_namespace_facts_unknown_effects_and_body_stability); + RUN_TEST(python_namespace_facts_parse_errors_are_complete_unknown); + RUN_TEST(python_namespace_facts_capture_and_copy_fail_atomically); + RUN_TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema); + RUN_TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload); + RUN_TEST(python_namespace_facts_unavailable_and_empty_survive_spill); + RUN_TEST(python_namespace_facts_multiple_bindings_and_unsupported_effects); + /* Wide-flat-file linearity (ms-typescript hang) */ RUN_TEST(extract_wide_flat_file_is_linear); #if defined(CBM_CALL_REFERENCE_LOOKUP_TEST_API) && CBM_CALL_REFERENCE_LOOKUP_TEST_API diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index f95adce16..70bbad8b4 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10884,6 +10884,337 @@ static int python_module_callers_pipeline_case(bool parallel) { PASS(); } +/* Insert after python_module_exact_lsp_edge. Parent registers the four TESTs. */ +static const char namespace_admission_control[] = "def target():\n" + " return 1\n" + "def keep():\n" + " return target()\n"; + +static const char *const namespace_admission_env_keys[] = {"CBM_WORKERS", + "CBM_INDEX_SINGLE_THREAD"}; + +typedef struct { + char root[512]; + char db[768]; + char *saved[2]; + bool created; + bool saved_env; +} namespace_admission_fixture; + +typedef struct { + int rc; + int errors; + int format; + cbm_incremental_route_t route; + bool migrated; + bool healthy; + char project[512]; + char *json; +} namespace_admission_result; + +static bool namespace_admission_begin(namespace_admission_fixture *f) { + int n = snprintf(f->root, sizeof(f->root), "%s/cbm-namespace-admission-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(f->root) || !cbm_mkdtemp(f->root)) + return false; + f->created = true; + n = snprintf(f->db, sizeof(f->db), "%s/namespace.db", f->root); + if (n <= 0 || (size_t)n >= sizeof(f->db)) + return false; + for (int i = 0; i < 2; i++) { + const char *old = getenv(namespace_admission_env_keys[i]); + f->saved[i] = old ? strdup(old) : NULL; + if (old && !f->saved[i]) + return false; + } + f->saved_env = true; + int env_rc = cbm_setenv(namespace_admission_env_keys[0], "1", 1); + env_rc |= cbm_setenv(namespace_admission_env_keys[1], "1", 1); + return env_rc == 0 && + python_module_write_fixture_file(f->root, "control.py", namespace_admission_control); +} + +static bool namespace_admission_finish(namespace_admission_fixture *f) { + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + if (f->saved_env) + restore_rc |= f->saved[i] ? cbm_setenv(namespace_admission_env_keys[i], f->saved[i], 1) + : cbm_unsetenv(namespace_admission_env_keys[i]); + free(f->saved[i]); + } + cbm_pipeline_incremental_test_reset_faults(); + int cleanup_rc = f->created ? th_rmtree(f->root) : 0; + return restore_rc == 0 && cleanup_rc == 0; +} + +static bool namespace_admission_exact_hash(cbm_store_t *store, const char *project, + const char *path, const char *source) { + cbm_file_hash_t row = {0}; + char expected[CBM_SHA256_HEX_LEN + 1]; + cbm_sha256_hex(source, strlen(source), expected); + bool correct = cbm_store_get_file_hash(store, project, path, &row) == CBM_STORE_OK && + row.sha256 && strcmp(row.sha256, expected) == 0; + cbm_store_clear_file_hash(&row); + return correct; +} + +static namespace_admission_result namespace_admission_run(namespace_admission_fixture *f, + const char *watch, const char *source, + int expected_files) { + namespace_admission_result result = { + .rc = -1, .errors = -1, .format = -1, .route = CBM_INCREMENTAL_ROUTE_NONE}; + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *pipeline = cbm_pipeline_new(f->root, f->db, CBM_MODE_FULL); + if (!pipeline) + return result; + result.rc = cbm_pipeline_run(pipeline); + result.route = cbm_pipeline_incremental_test_last_route(); + result.migrated = cbm_pipeline_had_format_migration(pipeline); + cbm_file_error_t *errors = NULL; + cbm_pipeline_get_file_errors(pipeline, &errors, &result.errors); + const char *project = cbm_pipeline_project_name(pipeline); + int n = project ? snprintf(result.project, sizeof(result.project), "%s", project) : -1; + cbm_store_t *store = result.rc == 0 && n > 0 && (size_t)n < sizeof(result.project) + ? cbm_store_open_path(f->db) + : NULL; + if (store) { + cbm_node_t *files = NULL; + int file_count = 0; + bool file_query = cbm_store_find_nodes_by_label(store, project, "File", &files, + &file_count) == CBM_STORE_OK; + int watched_files = 0; + for (int i = 0; file_query && i < file_count; i++) + if (files[i].file_path && strcmp(files[i].file_path, watch) == 0) + watched_files++; + cbm_store_free_nodes(files, file_count); + bool positive = python_module_exact_lsp_edge(store, project, "control.keep", "Function", + "control.target", "Function"); + bool hashes = namespace_admission_exact_hash(store, project, "control.py", + namespace_admission_control) && + namespace_admission_exact_hash(store, project, watch, source); + bool format_ok = cbm_store_get_format_version(store, &result.format) == CBM_STORE_OK; + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + bool row_query = + cbm_store_get_lsp_surfaces(store, project, &rows, &row_count) == CBM_STORE_OK; + int matches = 0; + for (int i = 0; row_query && i < row_count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, watch) != 0) + continue; + matches++; + if (rows[i].defs_json && !result.json) + result.json = strdup(rows[i].defs_json); + } + cbm_store_free_lsp_surfaces(rows, row_count); + bool python_row = matches == 1 && result.json && + cbm_lsp_surface_python_change(result.json, result.json) == 0 && + cbm_lsp_surface_python_change(result.json, NULL) == 1; + result.healthy = result.errors == 0 && file_query && file_count == expected_files && + watched_files == 1 && positive && hashes && format_ok && + result.format == CBM_INDEX_FORMAT_VERSION && row_query && python_row; + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + return result; +} + +TEST(pipeline_python_namespace_admission_upgrade_then_noop) { + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + namespace_admission_result first = {0}, upgraded = {0}, unchanged = {0}; + bool stamped = false; + if (setup) { + first = namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + cbm_store_t *store = first.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + int old_format = -1; + stamped = cbm_store_set_format_version(store, 1) == CBM_STORE_OK && + cbm_store_get_format_version(store, &old_format) == CBM_STORE_OK && + old_format == 1; + cbm_store_close(store); + } + if (stamped) + upgraded = + namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + if (upgraded.healthy) + unchanged = + namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + } + bool stable = first.json && upgraded.json && unchanged.json && + cbm_lsp_surface_python_change(first.json, upgraded.json) == 0 && + cbm_lsp_surface_python_change(upgraded.json, unchanged.json) == 0; + free(first.json); + free(upgraded.json); + free(unchanged.json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && stamped && cleaned); + ASSERT_TRUE(first.healthy && upgraded.healthy && unchanged.healthy && stable); + ASSERT_TRUE(upgraded.migrated); + ASSERT_FALSE(unchanged.migrated); + ASSERT_EQ(unchanged.route, CBM_INCREMENTAL_ROUTE_NOOP); + PASS(); +} + +TEST(pipeline_python_namespace_admission_body_alias_and_all) { + static const char initial[] = "from math import floor as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 1\n"; + static const char body_edit[] = "from math import floor as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 2\n"; + static const char alias_edit[] = "from math import ceil as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 2\n"; + static const char all_edit[] = "from math import ceil as Alias\n" + "__all__ = []\n" + "def body():\n return 2\n"; + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + namespace_admission_result phases[4] = {0}; + const char *sources[] = {initial, body_edit, alias_edit, all_edit}; + bool wrote_all = setup; + for (int i = 0; i < 4 && wrote_all; i++) { + wrote_all = python_module_write_fixture_file(fixture.root, "subject.py", sources[i]); + if (wrote_all) + phases[i] = namespace_admission_run(&fixture, "subject.py", sources[i], 2); + if (!phases[i].healthy) + break; + } + bool all_healthy = + phases[0].healthy && phases[1].healthy && phases[2].healthy && phases[3].healthy; + bool comparisons = all_healthy && + cbm_lsp_surface_python_change(phases[0].json, phases[1].json) == 0 && + cbm_lsp_surface_python_change(phases[1].json, phases[2].json) == 1 && + cbm_lsp_surface_python_change(phases[2].json, phases[3].json) == 1; + for (int i = 0; i < 4; i++) + free(phases[i].json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && wrote_all && cleaned && all_healthy && comparisons); + ASSERT_EQ(phases[1].route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(phases[2].route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_EQ(phases[3].route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + +TEST(pipeline_python_namespace_admission_deleted_and_empty) { + const char *sources[] = {"def removed():\n return 3\n", "# empty namespace\n"}; + bool setups[2] = {false, false}, removed[2] = {false, false}; + bool healthy[2] = {false, false}, absent[2] = {false, false}, cleaned[2] = {false, false}; + cbm_incremental_route_t routes[2] = {CBM_INCREMENTAL_ROUTE_NONE, CBM_INCREMENTAL_ROUTE_NONE}; + for (int i = 0; i < 2; i++) { + namespace_admission_fixture fixture = {0}; + setups[i] = namespace_admission_begin(&fixture); + if (setups[i]) + setups[i] = python_module_write_fixture_file(fixture.root, "gone.py", sources[i]); + namespace_admission_result before = {0}, after = {0}; + if (setups[i]) + before = namespace_admission_run(&fixture, "gone.py", sources[i], 2); + if (before.healthy) { + char path[768]; + int n = snprintf(path, sizeof(path), "%s/gone.py", fixture.root); + removed[i] = n > 0 && (size_t)n < sizeof(path) && cbm_unlink(path) == 0; + } + if (removed[i]) + after = namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + healthy[i] = before.healthy && after.healthy; + routes[i] = after.route; + cbm_store_t *store = after.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_node_t *nodes = NULL; + int count = 0; + absent[i] = cbm_store_find_nodes_by_file(store, after.project, "gone.py", &nodes, + &count) == CBM_STORE_OK && + count == 0; + cbm_store_free_nodes(nodes, count); + cbm_store_close(store); + } + free(before.json); + free(after.json); + cleaned[i] = namespace_admission_finish(&fixture); + } + for (int i = 0; i < 2; i++) { + ASSERT_TRUE(setups[i] && removed[i] && healthy[i] && absent[i] && cleaned[i]); + ASSERT_EQ(routes[i], CBM_INCREMENTAL_ROUTE_FORCED_FULL); + } + PASS(); +} + +TEST(pipeline_python_namespace_admission_invalid_equal_hash) { + static const char initial[] = "def body():\n return 1\n"; + static const char edited[] = "def body():\n return 2\n"; + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + if (setup) + setup = python_module_write_fixture_file(fixture.root, "subject.py", initial); + namespace_admission_result before = {0}, after = {0}; + bool corrupted = false, wrote = false; + char *old_sha = NULL, *new_sha = NULL; + if (setup) + before = namespace_admission_run(&fixture, "subject.py", initial, 2); + cbm_store_t *store = before.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_lsp_surface_row_t *rows = NULL; + int count = 0; + int rc = cbm_store_get_lsp_surfaces(store, before.project, &rows, &count); + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, "subject.py") != 0) + continue; + old_sha = rows[i].surface_sha ? strdup(rows[i].surface_sha) : NULL; + cbm_lsp_surface_row_t invalid = rows[i]; + invalid.defs_json = "{}"; /* Leave the old hash intact: validate before equality. */ + corrupted = old_sha && cbm_lsp_surface_python_change("{}", "{}") == -1 && + cbm_store_upsert_lsp_surface_batch(store, &invalid, 1) == CBM_STORE_OK; + break; + } + cbm_store_free_lsp_surfaces(rows, count); + rows = NULL; + count = 0; + bool confirmed = false; + rc = corrupted ? cbm_store_get_lsp_surfaces(store, before.project, &rows, &count) + : CBM_STORE_ERR; + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, "subject.py") != 0) + continue; + confirmed = rows[i].defs_json && strcmp(rows[i].defs_json, "{}") == 0 && + rows[i].surface_sha && strcmp(rows[i].surface_sha, old_sha) == 0; + break; + } + corrupted = corrupted && confirmed; + cbm_store_free_lsp_surfaces(rows, count); + cbm_store_close(store); + } + if (corrupted) + wrote = python_module_write_fixture_file(fixture.root, "subject.py", edited); + if (wrote) + after = namespace_admission_run(&fixture, "subject.py", edited, 2); + store = after.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_lsp_surface_row_t *rows = NULL; + int count = 0; + int rc = cbm_store_get_lsp_surfaces(store, after.project, &rows, &count); + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (rows[i].rel_path && strcmp(rows[i].rel_path, "subject.py") == 0 && + rows[i].surface_sha) { + new_sha = strdup(rows[i].surface_sha); + break; + } + } + cbm_store_free_lsp_surfaces(rows, count); + cbm_store_close(store); + } + bool equal_hash = old_sha && new_sha && strcmp(old_sha, new_sha) == 0; + bool equal_valid_surface = before.healthy && after.healthy && + cbm_lsp_surface_python_change(before.json, after.json) == 0; + free(old_sha); + free(new_sha); + free(before.json); + free(after.json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && corrupted && wrote && cleaned); + ASSERT_TRUE(before.healthy && after.healthy && equal_hash && equal_valid_surface); + ASSERT_EQ(after.route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + TEST(pipeline_python_module_callers_sequential) { return python_module_callers_pipeline_case(false); } @@ -18732,6 +19063,247 @@ TEST(pipeline_seq_ts_cross_uses_shared_registry) { PASS(); } +/* Build a real extraction surface without publishing a database. The returned + * JSON outlives both the extraction and collector arenas. */ +static char *pipeline_namespace_surface(const char *source, bool include_defs) { + CBMFileResult *fr = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "namespace_probe", "pkg/__init__.py", 0, NULL, NULL); + if (!fr || fr->has_error || fr->parse_incomplete) { + cbm_free_result(fr); + return NULL; + } + CBMArena arena; + cbm_arena_init(&arena); + CBMFileResult *cache[] = {fr}; + cbm_file_info_t files[] = {{.rel_path = "pkg/__init__.py", .language = CBM_LANG_PYTHON}}; + char *modules[] = {NULL}; + int starts[2] = {0, 0}; + int def_count = 0; + CBMLSPDef *defs = include_defs + ? cbm_pxc_collect_all_defs(NULL, &arena, cache, files, 1, + "namespace_probe", modules, &def_count, starts) + : NULL; + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + char *json = NULL; + if (cbm_lsp_surface_build_rows(NULL, "namespace_probe", cache, files, 1, defs, starts, &rows, + &row_count) == 0 && + row_count == 1 && rows[0].defs_json) { + json = strdup(rows[0].defs_json); + } + cbm_store_free_lsp_surfaces(rows, row_count); + free(defs); + free(modules[0]); + cbm_arena_destroy(&arena); + cbm_free_result(fr); + return json; +} + +TEST(pipeline_python_namespace_change_admission) { + const char *sources[] = { + "def target(value):\n return 1\n", + "def target(value):\n return 2\n", + "def target(value, extra):\n return 1\n", + "from first import Target as Alias\n", + "from second import Target as Alias\n", + "from first import Target as Alias\nfrom second import Target as Alias\n", + "from second import Target as Alias\nfrom first import Target as Alias\n", + "__all__ = []\n", + "__all__ = ['Alias']\n", + "__all__ = dynamic_names()\n", + "# empty namespace\n", + }; + enum { count = (int)(sizeof(sources) / sizeof(sources[0])) }; + char *json[count]; + bool complete = true; + for (int i = 0; i < count; i++) { + json[i] = pipeline_namespace_surface(sources[i], true); + complete = complete && json[i] != NULL; + } + int body = -9, signature = -9, alias = -9, order = -9; + int empty = -9, literal = -9, unknown = -9, deletion = -9, transition = -9; + const char *non_python = "{\"v\":2,\"lang\":0,\"lsp\":[],\"reg\":[],\"py\":null}"; + if (complete) { + body = cbm_lsp_surface_python_change(json[0], json[1]); + signature = cbm_lsp_surface_python_change(json[0], json[2]); + alias = cbm_lsp_surface_python_change(json[3], json[4]); + order = cbm_lsp_surface_python_change(json[5], json[6]); + empty = cbm_lsp_surface_python_change(json[10], json[7]); + literal = cbm_lsp_surface_python_change(json[7], json[8]); + unknown = cbm_lsp_surface_python_change(json[7], json[9]); + deletion = cbm_lsp_surface_python_change(json[10], NULL); + transition = cbm_lsp_surface_python_change(non_python, json[10]); + } + for (int i = 0; i < count; i++) { + free(json[i]); + } + ASSERT_TRUE(complete); + ASSERT_EQ(body, 0); + ASSERT_EQ(signature, 1); + ASSERT_EQ(alias, 1); + ASSERT_EQ(order, 1); + ASSERT_EQ(empty, 1); + ASSERT_EQ(literal, 1); + ASSERT_EQ(unknown, 1); + ASSERT_EQ(deletion, 1); + ASSERT_EQ(transition, 1); + ASSERT_EQ(cbm_lsp_surface_python_change(non_python, NULL), 0); + ASSERT_EQ(cbm_lsp_surface_python_change(non_python, non_python), 0); + PASS(); +} + +static char *pipeline_namespace_payload_json(const char *json) { + yyjson_doc *doc = json ? yyjson_read(json, strlen(json), 0) : NULL; + if (!doc) { + return NULL; + } + yyjson_val *payload = yyjson_obj_get(yyjson_doc_get_root(doc), "py"); + char *copy = payload ? yyjson_val_write(payload, 0, NULL) : NULL; + yyjson_doc_free(doc); + return copy; +} + +TEST(pipeline_python_namespace_codec_owns_ordered_facts) { + const char *source = "def plain(value):\n return value\n" + "from .one import First as Alias\n" + "from .two import Second as Alias\n" + "__all__ = ['Alias', '_private', '', 'odd|name']\n" + "__all__ += ('tail',)\n" + "__all__ = []\n" + "__all__ = dynamic_names()\n"; + char *full = pipeline_namespace_surface(source, true); + char *probe = pipeline_namespace_surface(source, false); + char *full_facts = pipeline_namespace_payload_json(full); + char *probe_facts = pipeline_namespace_payload_json(probe); + bool parity = full_facts && probe_facts && strcmp(full_facts, probe_facts) == 0; + bool distinct_defs = full && probe && strcmp(full, probe) != 0; + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t copied = {0}, failed = {0}; + cbm_py_namespace_test_copy_fail_after(0); + int failed_rc = cbm_lsp_surface_python_namespace_from_json(&arena, full, &failed); + cbm_py_namespace_test_copy_fail_after(-1); + int rc = cbm_lsp_surface_python_namespace_from_json(&arena, full, &copied); + free(full); + free(probe); + free(full_facts); + free(probe_facts); + + /* The original result, collector, JSON documents and serialized bytes are + * all gone before the copied strings and nested arrays are inspected. */ + bool origin = rc == 1 && copied.module_qn && copied.rel_path && + strcmp(copied.module_qn, "namespace_probe.pkg.__init__") == 0 && + strcmp(copied.rel_path, "pkg/__init__.py") == 0; + bool valid = rc == 1 && cbm_py_namespace_facts_valid(&copied.facts) && + copied.facts.status == CBM_PY_NS_COMPLETE && + copied.facts.language == CBM_LANG_PYTHON; + int imports = 0; + bool import_order = true, names = false, append = false, empty = false, unknown = false; + if (valid) { + for (int i = 0; i < copied.facts.count; i++) { + const CBMPyNamespaceFact *f = &copied.facts.items[i]; + if (f->kind == CBM_PY_NS_IMPORT_NAME) { + const char *module = imports == 0 ? "one" : "two"; + const char *member = imports == 0 ? "First" : "Second"; + import_order = import_order && imports < 2 && f->relative_level == 1 && + f->module_name && strcmp(f->module_name, module) == 0 && + f->member_name && strcmp(f->member_name, member) == 0 && + f->local_name && strcmp(f->local_name, "Alias") == 0; + imports++; + } + if (f->kind == CBM_PY_NS_ALL_SET && f->name_count == 4) { + names = f->sequence_kind == CBM_PY_NS_SEQUENCE_LIST && + strcmp(f->names[0], "Alias") == 0 && strcmp(f->names[1], "_private") == 0 && + strcmp(f->names[2], "") == 0 && strcmp(f->names[3], "odd|name") == 0; + } + if (f->kind == CBM_PY_NS_ALL_APPEND) { + append = f->sequence_kind == CBM_PY_NS_SEQUENCE_TUPLE && f->name_count == 1 && + strcmp(f->names[0], "tail") == 0; + } + empty = + empty || (f->kind == CBM_PY_NS_ALL_SET && f->name_count == 0 && f->names == NULL); + unknown = unknown || f->kind == CBM_PY_NS_ALL_UNKNOWN; + } + } + bool failure_empty = failed.facts.items == NULL && failed.facts.count == 0 && + failed.module_qn == NULL && failed.rel_path == NULL; + cbm_arena_destroy(&arena); + ASSERT_TRUE(parity); + ASSERT_TRUE(distinct_defs); + ASSERT_EQ(failed_rc, -1); + ASSERT_TRUE(failure_empty); + ASSERT_EQ(rc, 1); + ASSERT_TRUE(origin); + ASSERT_TRUE(valid); + ASSERT_EQ(imports, 2); + ASSERT_TRUE(import_order); + ASSERT_TRUE(names); + ASSERT_TRUE(append); + ASSERT_TRUE(empty); + ASSERT_TRUE(unknown); + PASS(); +} + +TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows) { + /* Numeric values below are the frozen version-1 namespace wire values. + * Even an empty LSP array must validate its namespace envelope. */ + const char *bad[] = { + "{\"v\":1,\"lsp\":[]}", + "{\"v\":2,\"lang\":1,\"lsp\":[]}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"lang\":1.0,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"v\":2,\"lang\":0,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":3,\"failure\":2,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":99,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\\u0000suffix\",\"status\":2," + "\"failure\":0,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":999,\"d\":0,\"s\":0,\"r\":0,\"f\":0,\"l\":0," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":8,\"d\":0,\"s\":1,\"r\":0,\"f\":0,\"l\":4294967296," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}", + }; + bool rejected = true; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t facts = {0}; + CBMLSPDef *defs = NULL; + int ns_rc = cbm_lsp_surface_python_namespace_from_json(&arena, bad[i], &facts); + int defs_rc = cbm_lsp_surface_defs_from_json(&arena, bad[i], &defs); + int compare_rc = cbm_lsp_surface_python_change(bad[i], bad[i]); + int delete_rc = cbm_lsp_surface_python_change(bad[i], NULL); + rejected = rejected && ns_rc == -1 && defs_rc == -1 && compare_rc == -1 && + delete_rc == -1 && facts.facts.items == NULL && defs == NULL; + cbm_arena_destroy(&arena); + } + const char *unknown = + "{\"v\":2,\"lang\":1,\"lsp\":[],\"reg\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":7,\"d\":0,\"s\":0,\"r\":7,\"f\":0,\"l\":0," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}"; + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t facts = {0}; + int unknown_rc = cbm_lsp_surface_python_namespace_from_json(&arena, unknown, &facts); + int equal_rc = cbm_lsp_surface_python_change(unknown, unknown); + bool retained = unknown_rc == 1 && facts.facts.count == 1 && + facts.facts.items[0].kind == CBM_PY_NS_UNKNOWN && + facts.facts.items[0].reason == CBM_PY_NS_REASON_PARSE; + cbm_arena_destroy(&arena); + ASSERT_TRUE(rejected); + ASSERT_EQ(unknown_rc, 1); + ASSERT_EQ(equal_rc, 0); + ASSERT_TRUE(retained); + PASS(); +} + /* Object arrays are non-flat in yyjson. Exercise ordered decoding and arena * ownership across a large surface, including nested string arrays. */ TEST(pipeline_lsp_surface_large_object_array_decode) { @@ -18739,7 +19311,7 @@ TEST(pipeline_lsp_surface_large_object_array_decode) { const size_t capacity = (size_t)count * 192 + 64; char *json = malloc(capacity); ASSERT_NOT_NULL(json); - size_t used = (size_t)snprintf(json, capacity, "{\"v\":1,\"lsp\":["); + size_t used = (size_t)snprintf(json, capacity, "{\"v\":2,\"lang\":0,\"py\":null,\"lsp\":["); for (int i = 0; i < count; i++) { int n = snprintf(json + used, capacity - used, "%s{\"qn\":\"pkg.f%d\",\"sn\":\"f%d\",\"lb\":\"Function\"," @@ -18831,7 +19403,7 @@ TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant) { snprintf(sha_baseline, sizeof(sha_baseline), "%s", row_a->surface_sha); /* The row is the versioned codec envelope with the def present. */ snprintf(json_probe, sizeof(json_probe), "%.20s", row_a->defs_json); - ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":1") != NULL); + ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":2") != NULL); ASSERT_TRUE(strstr(row_a->defs_json, "surface_probe") != NULL); } else if (round == 1) { ASSERT_STR_EQ(row_a->surface_sha, sha_baseline); @@ -21120,6 +21692,13 @@ SUITE(pipeline) { RUN_TEST(pipeline_fixture_file_parent_is_preserved); RUN_TEST(pipeline_fixture_node_count_requires_exact_source_nodes); RUN_TEST(pipeline_nested_fixture_graph_has_endpoints); + RUN_TEST(pipeline_python_namespace_admission_upgrade_then_noop); + RUN_TEST(pipeline_python_namespace_admission_body_alias_and_all); + RUN_TEST(pipeline_python_namespace_admission_deleted_and_empty); + RUN_TEST(pipeline_python_namespace_admission_invalid_equal_hash); + RUN_TEST(pipeline_python_namespace_change_admission); + RUN_TEST(pipeline_python_namespace_codec_owns_ordered_facts); + RUN_TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows); RUN_TEST(pipeline_lsp_surface_large_object_array_decode); RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant); /* Index lock */