From f35745c79facb51ed857c442dcf9f177a963d595 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 9 May 2026 12:11:50 +0200 Subject: [PATCH] =?UTF-8?q?feat(py=5Flsp):=20with-as,=20except-as,=20tuple?= =?UTF-8?q?=20unpack,=20slice,=20container=20TypeVar=20special-cases=20?= =?UTF-8?q?=E2=80=94=20bench=20100%?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 8. Stress test suite (test_py_lsp_stress.c) added with 22 advanced patterns; bench fixture grew to 178 lines covering the full surface. with-as binding via __enter__: - Look up __enter__ on the context manager's NAMED type. Use its return type when known; fall back to the manager type itself (matches the practical case where __enter__ returns self). Handles tree-sitter Python's wrapping of `X as y` in an as_pattern node inside with_item, plus the flat field-only shape. except-as binding: - Bind exception alias to the caught type. Resolves through except_clause's as_pattern wrapper and the as_pattern_target identifier wrapper. Tuple unpacking: - Both `a, b = f()` and `for k, v in d.items()` destructure the RHS element-by-element when the rhs is TUPLE or TEMPLATE("tuple", ...). *rest binds the trailing slice as list[elem]. Shared helper py_bind_for_target used by for-statement, comprehension for_in_clause, and assignment LHS. Slice subscript: - `lst[1:3]` returns the same TEMPLATE container — list[T][1:3] is list[T], dict[K, V][1:3] is dict[K, V] (rare but consistent). Container TypeVar substitution: - Without a constraint solver we can't generally substitute K, V into registered method return types. Hand-rolled the most-impactful cases: dict.items/keys/values/get/pop/copy, list.copy/__iter__/pop, set.copy. These return properly parameterized ItemsView[K, V] / KeysView[K] / ValuesView[V] / Optional[V] / etc. so subsequent iteration and unpacking yield typed elements. ItemsView / KeysView / ValuesView / dict_items / dict_keys / dict_values added to py_iterable_element_type. ItemsView[K, V] iterates as tuple[K, V], unpacked correctly by py_bind_for_target. Bench: 178 LOC, 52 calls, **52 resolved (100%)**, 11.59 ms (ASan + UBSan). 22 new stress tests, 8 of 10 documented gaps closed in this round. Remaining gaps documented inline (typeddict_subscript, post-early-return narrowing, function-as-dict-value, match sequence pattern, lambda inference). All 2898 prior tests stay green; total 2920 / 0. --- Makefile.cbm | 4 +- internal/cbm/lsp/py_lsp.c | 315 ++++++++++++++++++++-- tests/test_main.c | 2 + tests/test_py_lsp_stress.c | 516 +++++++++++++++++++++++++++++++++++++ 4 files changed, 816 insertions(+), 21 deletions(-) create mode 100644 tests/test_py_lsp_stress.c diff --git a/Makefile.cbm b/Makefile.cbm index 28e4cc0e0..13f2d863e 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -317,6 +317,8 @@ TEST_PY_LSP_SRCS = tests/test_py_lsp.c TEST_PY_LSP_BENCH_SRCS = tests/test_py_lsp_bench.c +TEST_PY_LSP_STRESS_SRCS = tests/test_py_lsp_stress.c + TEST_INTEGRATION_SRCS = tests/test_integration.c tests/test_incremental.c TEST_TRACES_SRCS = tests/test_traces.c @@ -334,7 +336,7 @@ TEST_YAML_SRCS = tests/test_yaml.c TEST_SIMHASH_SRCS = tests/test_simhash.c -ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_INTEGRATION_SRCS) +ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_INTEGRATION_SRCS) # ── Build directories ──────────────────────────────────────────── diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 307bf57e1..e9ae6fcb8 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -32,6 +32,7 @@ static const CBMType* py_lookup_field(PyLSPContext* ctx, const char* type_qn, const char* field_name); static void py_register_instance_field(PyLSPContext* ctx, const char* class_qn, const char* field_name, const CBMType* field_type); +static void py_bind_for_target(PyLSPContext* ctx, TSNode left, const CBMType* elem_type); void py_lsp_init(PyLSPContext* ctx, CBMArena* arena, const char* source, int source_len, const CBMTypeRegistry* registry, const char* module_qn, CBMResolvedCallArray* out) { @@ -402,8 +403,52 @@ static const CBMType* py_func_return_type(PyLSPContext* ctx, const char* func_qn /* Element type of an iterable. For TEMPLATE("list"|"set"|"Iterable"|..., * [T]), return T. For TEMPLATE("dict", [K, V]) — `for x in d` iterates * keys, return K. For TEMPLATE("tuple", [A, B, ...]) return UNION(A, B, - * ...) since `for x in (a, b)` yields heterogeneous types. NAMED("list") - * with no template args -> UNKNOWN. */ + * ...) since `for x in (a, b)` yields heterogeneous types. dict_items / + * ItemsView return tuple[K, V] so unpacking works. NAMED("list") with + * no template args -> UNKNOWN. */ +/* Bind a `for X in ...` target. Supports identifier (single binding) and + * pattern_list / tuple_pattern (destructure tuple element type). */ +static void py_bind_for_target(PyLSPContext* ctx, TSNode left, const CBMType* elem_type) { + if (ts_node_is_null(left) || !ctx) return; + const char* lk = ts_node_type(left); + if (strcmp(lk, "identifier") == 0) { + char* name = py_node_text(ctx, left); + if (name) cbm_scope_bind(ctx->current_scope, name, elem_type); + return; + } + if (strcmp(lk, "pattern_list") == 0 || strcmp(lk, "tuple_pattern") == 0 || + strcmp(lk, "list_pattern") == 0) { + // Destructure tuple element type element-by-element. + const CBMType* const* elems = NULL; + int count = 0; + if (elem_type) { + if (elem_type->kind == CBM_TYPE_TUPLE) { + elems = elem_type->data.tuple.elems; + count = elem_type->data.tuple.count; + } else if (elem_type->kind == CBM_TYPE_TEMPLATE && + elem_type->data.template_type.template_name && + strcmp(elem_type->data.template_type.template_name, "tuple") == 0) { + elems = elem_type->data.template_type.template_args; + count = elem_type->data.template_type.arg_count; + } + } + uint32_t lc = ts_node_named_child_count(left); + for (uint32_t i = 0; i < lc; i++) { + TSNode tgt = ts_node_named_child(left, i); + if (ts_node_is_null(tgt)) continue; + const char* tk = ts_node_type(tgt); + if (strcmp(tk, "identifier") == 0) { + char* nm = py_node_text(ctx, tgt); + if (!nm) continue; + const CBMType* bind_type; + if (elems && (int)i < count && elems[i]) bind_type = elems[i]; + else bind_type = cbm_type_unknown(); + cbm_scope_bind(ctx->current_scope, nm, bind_type); + } + } + } +} + static const CBMType* py_iterable_element_type(PyLSPContext* ctx, const CBMType* iter_type) { if (!iter_type) return cbm_type_unknown(); if (iter_type->kind == CBM_TYPE_TEMPLATE) { @@ -416,10 +461,21 @@ static const CBMType* py_iterable_element_type(PyLSPContext* ctx, const CBMType* strcmp(name, "Iterator") == 0 || strcmp(name, "Sequence") == 0 || strcmp(name, "MutableSequence") == 0 || strcmp(name, "AsyncIterable") == 0 || strcmp(name, "AsyncIterator") == 0 || strcmp(name, "Generator") == 0 || + strcmp(name, "AsyncGenerator") == 0 || strcmp(name, "Reversible") == 0 || strcmp(name, "Collection") == 0 || - strcmp(name, "Container") == 0 || strcmp(name, "deque") == 0) { + strcmp(name, "Container") == 0 || strcmp(name, "deque") == 0 || + strcmp(name, "KeysView") == 0 || strcmp(name, "ValuesView") == 0 || + strcmp(name, "dict_keys") == 0 || strcmp(name, "dict_values") == 0) { return args[0]; } + if (strcmp(name, "ItemsView") == 0 || strcmp(name, "dict_items") == 0) { + // Iterating ItemsView[K, V] yields tuple[K, V]. + if (n >= 2) { + const CBMType* elems[3] = { args[0], args[1], NULL }; + return cbm_type_tuple(ctx->arena, elems, 2); + } + return cbm_type_unknown(); + } if (strcmp(name, "dict") == 0 || strcmp(name, "Mapping") == 0 || strcmp(name, "MutableMapping") == 0 || strcmp(name, "defaultdict") == 0 || strcmp(name, "OrderedDict") == 0) { @@ -567,6 +623,66 @@ static const CBMType* py_eval_expr_type(PyLSPContext* ctx, TSNode node) { if (ts_node_is_null(fn)) return cbm_type_unknown(); const char* fk = ts_node_type(fn); + // Container method special-cases: dict.items() / .keys() / .values(), + // list.copy(), set.copy() etc. Without a constraint solver we can't + // substitute K, V into the registered method's return type, so we + // hand-roll the most-impactful container methods. + if (strcmp(fk, "attribute") == 0) { + TSNode obj = ts_node_child_by_field_name(fn, "object", 6); + TSNode attr = ts_node_child_by_field_name(fn, "attribute", 9); + if (!ts_node_is_null(obj) && !ts_node_is_null(attr)) { + const CBMType* obj_type = py_eval_expr_type(ctx, obj); + char* mname = py_node_text(ctx, attr); + if (mname && obj_type && obj_type->kind == CBM_TYPE_TEMPLATE) { + const char* tname = obj_type->data.template_type.template_name; + const CBMType** args = obj_type->data.template_type.template_args; + int n = obj_type->data.template_type.arg_count; + bool is_dict_like = tname && (strcmp(tname, "dict") == 0 || + strcmp(tname, "Mapping") == 0 || + strcmp(tname, "MutableMapping") == 0 || + strcmp(tname, "defaultdict") == 0 || + strcmp(tname, "OrderedDict") == 0); + bool is_list_like = tname && (strcmp(tname, "list") == 0 || + strcmp(tname, "set") == 0 || + strcmp(tname, "frozenset") == 0 || + strcmp(tname, "deque") == 0); + if (is_dict_like && args && n >= 2) { + if (strcmp(mname, "items") == 0) { + const CBMType* pair[3] = { args[0], args[1], NULL }; + return cbm_type_template(ctx->arena, "ItemsView", pair, 2); + } + if (strcmp(mname, "keys") == 0) { + const CBMType* k1[2] = { args[0], NULL }; + return cbm_type_template(ctx->arena, "KeysView", k1, 1); + } + if (strcmp(mname, "values") == 0) { + const CBMType* v1[2] = { args[1], NULL }; + return cbm_type_template(ctx->arena, "ValuesView", v1, 1); + } + if (strcmp(mname, "get") == 0) { + // dict.get(k) -> Optional[V] + return cbm_type_optional(ctx->arena, args[1]); + } + if (strcmp(mname, "pop") == 0) { + return args[1]; + } + if (strcmp(mname, "copy") == 0) { + return obj_type; // dict[K, V] + } + } + if (is_list_like && args && n >= 1) { + if (strcmp(mname, "copy") == 0 || + strcmp(mname, "__iter__") == 0) { + return obj_type; + } + if (strcmp(mname, "pop") == 0) { + return args[0]; + } + } + } + } + } + // typing.cast(T, x) -> NAMED(T). typing.assert_type(x, T) -> type-of(x). // Detect by the call's function expression: matches either bare `cast` / // `assert_type` (when imported from typing) or `typing.cast` style. @@ -736,6 +852,15 @@ static const CBMType* py_eval_expr_type(PyLSPContext* ctx, TSNode node) { const CBMType** args = container->data.template_type.template_args; int n = container->data.template_type.arg_count; if (!tname || !args || n <= 0) return cbm_type_unknown(); + + // Slice subscript returns the same container type: + // list[T][1:3] -> list[T]; str[1:3] -> str (handled via BUILTIN + // path elsewhere). + TSNode sub = ts_node_child_by_field_name(node, "subscript", 9); + if (!ts_node_is_null(sub) && strcmp(ts_node_type(sub), "slice") == 0) { + return container; + } + if (strcmp(tname, "dict") == 0 || strcmp(tname, "Mapping") == 0 || strcmp(tname, "MutableMapping") == 0 || strcmp(tname, "defaultdict") == 0 || strcmp(tname, "OrderedDict") == 0 || strcmp(tname, "ChainMap") == 0) { @@ -750,8 +875,6 @@ static const CBMType* py_eval_expr_type(PyLSPContext* ctx, TSNode node) { return args[0]; } if (strcmp(tname, "tuple") == 0) { - // Try to index by literal integer subscript. - TSNode sub = ts_node_child_by_field_name(node, "subscript", 9); if (!ts_node_is_null(sub) && strcmp(ts_node_type(sub), "integer") == 0) { char* idx_text = py_node_text(ctx, sub); if (idx_text) { @@ -759,7 +882,6 @@ static const CBMType* py_eval_expr_type(PyLSPContext* ctx, TSNode node) { if (idx >= 0 && idx < n) return args[idx]; } } - // Non-literal index: union of all elements. if (n == 1) return args[0]; return cbm_type_union(ctx->arena, args, n); } @@ -796,6 +918,64 @@ static void py_process_statement(PyLSPContext* ctx, TSNode node) { if (ts_node_is_null(left)) return; const char* lk = ts_node_type(left); + // Tuple/list pattern unpacking: a, b = f() / [a, b] = f() / + // (a, b) = f(). Each LHS element binds to the corresponding + // element of the RHS tuple. *rest binds the remaining elements + // as a list (best-effort: list of element type). + if (strcmp(lk, "pattern_list") == 0 || strcmp(lk, "tuple_pattern") == 0 || + strcmp(lk, "list_pattern") == 0 || strcmp(lk, "expression_list") == 0) { + uint32_t lc = ts_node_named_child_count(left); + // Collect RHS element types. + const CBMType* const* rhs_elems = NULL; + int rhs_count = 0; + if (rhs_type) { + if (rhs_type->kind == CBM_TYPE_TUPLE) { + rhs_elems = rhs_type->data.tuple.elems; + rhs_count = rhs_type->data.tuple.count; + } else if (rhs_type->kind == CBM_TYPE_TEMPLATE && + rhs_type->data.template_type.template_name) { + const char* tn = rhs_type->data.template_type.template_name; + if (strcmp(tn, "tuple") == 0) { + rhs_elems = rhs_type->data.template_type.template_args; + rhs_count = rhs_type->data.template_type.arg_count; + } + } + } + // Element type for star-rest binding (best-effort element of + // the iterable; for a pure tuple use UNKNOWN since the rest + // is heterogeneous). + const CBMType* elem_type = py_iterable_element_type(ctx, rhs_type); + + for (uint32_t i = 0; i < lc; i++) { + TSNode tgt = ts_node_named_child(left, i); + if (ts_node_is_null(tgt)) continue; + const char* tk = ts_node_type(tgt); + bool is_rest = false; + TSNode bind_target = tgt; + if (strcmp(tk, "list_splat_pattern") == 0 || + strcmp(tk, "list_splat") == 0) { + is_rest = true; + if (ts_node_named_child_count(tgt) > 0) { + bind_target = ts_node_named_child(tgt, 0); + } + } + if (strcmp(ts_node_type(bind_target), "identifier") != 0) continue; + char* nm = py_node_text(ctx, bind_target); + if (!nm) continue; + const CBMType* bind_type; + if (is_rest) { + bind_type = elem_type + ? cbm_type_template(ctx->arena, "list", &elem_type, 1) + : cbm_type_unknown(); + } else if (rhs_elems && (int)i < rhs_count && rhs_elems[i]) { + bind_type = rhs_elems[i]; + } else { + bind_type = elem_type ? elem_type : cbm_type_unknown(); + } + cbm_scope_bind(ctx->current_scope, nm, bind_type); + } + return; + } if (strcmp(lk, "identifier") == 0) { char* name = py_node_text(ctx, left); if (name) cbm_scope_bind(ctx->current_scope, name, rhs_type); @@ -828,16 +1008,16 @@ static void py_process_statement(PyLSPContext* ctx, TSNode node) { const CBMType* iter_type = py_eval_expr_type(ctx, right); elem_type = py_iterable_element_type(ctx, iter_type); } - if (!ts_node_is_null(left) && strcmp(ts_node_type(left), "identifier") == 0) { - char* name = py_node_text(ctx, left); - if (name) cbm_scope_bind(ctx->current_scope, name, elem_type); - } + py_bind_for_target(ctx, left, elem_type); return; } if (strcmp(k, "with_statement") == 0) { - // with X as y: bind y to type-of(X.__enter__()) - // For now bind y to UNKNOWN — Phase 6+ refines. + // with X as y: bind y to X.__enter__() return type. Tree-sitter + // Python wraps `X as y` in an `as_pattern` whose first named child + // is X (value) and second is the target identifier `y` (alias). + // The with_item may either contain the as_pattern as its child, or + // directly expose value/alias as field children — we handle both. uint32_t nc = ts_node_named_child_count(node); for (uint32_t i = 0; i < nc; i++) { TSNode child = ts_node_named_child(node, i); @@ -846,10 +1026,109 @@ static void py_process_statement(PyLSPContext* ctx, TSNode node) { for (uint32_t j = 0; j < cn; j++) { TSNode item = ts_node_named_child(child, j); if (strcmp(ts_node_type(item), "with_item") != 0) continue; + + // Try field lookup first; fall back to as_pattern walk. + TSNode value = ts_node_child_by_field_name(item, "value", 5); TSNode alias = ts_node_child_by_field_name(item, "alias", 5); - if (!ts_node_is_null(alias) && strcmp(ts_node_type(alias), "identifier") == 0) { - char* name = py_node_text(ctx, alias); - if (name) cbm_scope_bind(ctx->current_scope, name, cbm_type_unknown()); + + if (ts_node_is_null(value) || ts_node_is_null(alias)) { + // Look for an as_pattern child. + uint32_t ic = ts_node_named_child_count(item); + for (uint32_t k2 = 0; k2 < ic; k2++) { + TSNode c = ts_node_named_child(item, k2); + if (strcmp(ts_node_type(c), "as_pattern") == 0) { + uint32_t ac = ts_node_named_child_count(c); + if (ac >= 1) value = ts_node_named_child(c, 0); + if (ac >= 2) alias = ts_node_named_child(c, 1); + // Field-based lookup as another option. + if (ts_node_is_null(alias)) { + alias = ts_node_child_by_field_name(c, "alias", 5); + } + break; + } + } + } + + if (ts_node_is_null(alias) || + strcmp(ts_node_type(alias), "identifier") != 0) { + // alias may itself be an as_pattern_target wrapping an + // identifier; walk through it. + if (!ts_node_is_null(alias) && + ts_node_named_child_count(alias) > 0) { + TSNode inner = ts_node_named_child(alias, 0); + if (strcmp(ts_node_type(inner), "identifier") == 0) { + alias = inner; + } + } + } + if (ts_node_is_null(alias) || + strcmp(ts_node_type(alias), "identifier") != 0) continue; + char* name = py_node_text(ctx, alias); + if (!name) continue; + const CBMType* cm_type = ts_node_is_null(value) + ? cbm_type_unknown() : py_eval_expr_type(ctx, value); + const CBMType* bind_type = cm_type; + if (cm_type && cm_type->kind == CBM_TYPE_NAMED) { + const CBMRegisteredFunc* enter = py_lookup_attribute(ctx, + cm_type->data.named.qualified_name, "__enter__"); + if (enter) { + const CBMType* ret = py_func_return_type_recv(ctx, + enter->qualified_name, + cm_type->data.named.qualified_name); + if (ret && !cbm_type_is_unknown(ret)) { + bind_type = ret; + } + } + } + cbm_scope_bind(ctx->current_scope, name, bind_type); + } + } + return; + } + + if (strcmp(k, "try_statement") == 0) { + // except E as e: bind e to NAMED(E). Tree-sitter Python wraps + // `E as e` either as an as_pattern child of the except_clause, + // or as flat children. Handle both shapes. + uint32_t nc = ts_node_named_child_count(node); + for (uint32_t i = 0; i < nc; i++) { + TSNode clause = ts_node_named_child(node, i); + if (strcmp(ts_node_type(clause), "except_clause") != 0) continue; + + TSNode exc_type = {0}; + TSNode exc_name = {0}; + uint32_t ccn = ts_node_named_child_count(clause); + for (uint32_t j = 0; j < ccn; j++) { + TSNode c = ts_node_named_child(clause, j); + const char* ck = ts_node_type(c); + if (strcmp(ck, "block") == 0) continue; + if (strcmp(ck, "as_pattern") == 0) { + uint32_t ac = ts_node_named_child_count(c); + if (ac >= 1) exc_type = ts_node_named_child(c, 0); + if (ac >= 2) exc_name = ts_node_named_child(c, 1); + // alias may be wrapped in as_pattern_target + if (!ts_node_is_null(exc_name)) { + const char* nk = ts_node_type(exc_name); + if (strcmp(nk, "as_pattern_target") == 0 && + ts_node_named_child_count(exc_name) > 0) { + exc_name = ts_node_named_child(exc_name, 0); + } + } + continue; + } + if (ts_node_is_null(exc_type)) exc_type = c; + else if (ts_node_is_null(exc_name) && + strcmp(ck, "identifier") == 0) { + exc_name = c; + } + } + if (!ts_node_is_null(exc_name) && !ts_node_is_null(exc_type) && + strcmp(ts_node_type(exc_name), "identifier") == 0) { + char* nm = py_node_text(ctx, exc_name); + char* tn = py_node_text(ctx, exc_type); + if (nm && tn) { + cbm_scope_bind(ctx->current_scope, nm, + py_resolve_annotation(ctx, tn)); } } } @@ -1322,11 +1601,7 @@ static void py_resolve_calls_in(PyLSPContext* ctx, TSNode node) { const CBMType* iter_type = py_eval_expr_type(ctx, right); elem_type = py_iterable_element_type(ctx, iter_type); } - if (!ts_node_is_null(left) && - strcmp(ts_node_type(left), "identifier") == 0) { - char* name = py_node_text(ctx, left); - if (name) cbm_scope_bind(ctx->current_scope, name, elem_type); - } + py_bind_for_target(ctx, left, elem_type); } } // Second pass: recurse into all children (body + filter clauses diff --git a/tests/test_main.c b/tests/test_main.c index 5f5c27e9c..75b9d6ef3 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -45,6 +45,7 @@ extern void suite_scope(void); extern void suite_type_rep(void); extern void suite_py_lsp(void); extern void suite_py_lsp_bench(void); +extern void suite_py_lsp_stress(void); extern void suite_store_arch(void); extern void suite_store_bulk(void); extern void suite_traces(void); @@ -122,6 +123,7 @@ int main(void) { RUN_SUITE(c_lsp); RUN_SUITE(py_lsp); RUN_SUITE(py_lsp_bench); + RUN_SUITE(py_lsp_stress); /* Architecture + ADR + Louvain */ RUN_SUITE(store_arch); diff --git a/tests/test_py_lsp_stress.c b/tests/test_py_lsp_stress.c new file mode 100644 index 000000000..218fe702b --- /dev/null +++ b/tests/test_py_lsp_stress.c @@ -0,0 +1,516 @@ +/* + * test_py_lsp_stress.c — Advanced Python pattern probes. + * + * Each test covers one pattern Pyright resolves. Tests that pass + * confirm parity. Tests that fail are honest gaps — they get either + * fixed in a follow-up round or documented as deferred per + * docs/BENCHMARK_PYTHON.md. + * + * Suite groups: patterns by feature area (typing helpers, + * inheritance, control flow, containers, decorators, advanced + * constructs). + */ +#include "test_framework.h" +#include "cbm.h" +#include "lsp/py_lsp.h" + +static CBMFileResult *extract_py(const char *source) { + return cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "test", "main.py", 0, NULL, NULL); +} + +static int find_resolved(const CBMFileResult *r, const char *callerSub, const char *calleeSub) { + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + if (rc->caller_qn && strstr(rc->caller_qn, callerSub) && rc->callee_qn && + strstr(rc->callee_qn, calleeSub)) + return i; + } + return -1; +} + +__attribute__((unused)) +static int require_resolved(const CBMFileResult *r, const char *callerSub, const char *calleeSub) { + int idx = find_resolved(r, callerSub, calleeSub); + if (idx < 0) { + printf(" MISSING resolved call: caller~%s -> callee~%s (have %d)\n", + callerSub, calleeSub, r->resolved_calls.count); + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + printf(" %s -> %s [%s %.2f]\n", + rc->caller_qn ? rc->caller_qn : "(null)", + rc->callee_qn ? rc->callee_qn : "(null)", + rc->strategy ? rc->strategy : "(null)", + rc->confidence); + } + } + return idx; +} + +/* ── Typing helpers ─────────────────────────────────────────── */ + +TEST(stress_namedtuple_class_form) { + /* class Point(NamedTuple): x: int → Point(1, 2).x */ + CBMFileResult *r = extract_py( + "from typing import NamedTuple\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "class Pair(NamedTuple):\n" + " a: Foo\n" + " b: Foo\n" + "def use():\n" + " p = Pair(Foo(), Foo())\n" + " return p.a.method()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_typeddict_subscript) { + /* class TD(TypedDict): x: Foo. d: TD = ...; d["x"].method() */ + CBMFileResult *r = extract_py( + "from typing import TypedDict\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "class TD(TypedDict):\n" + " foo: Foo\n" + "def use(d: TD):\n" + " return d['foo'].method()\n"); + ASSERT_NOT_NULL(r); + /* TypedDict["literal"] should narrow to the field type — gap if fails */ + int idx = find_resolved(r, "use", "method"); + if (idx < 0) printf(" KNOWN GAP: TypedDict['literal'] narrowing\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_protocol_structural) { + /* def use(x: HasMethod): x.method() + * where HasMethod is a Protocol */ + CBMFileResult *r = extract_py( + "from typing import Protocol\n" + "class HasMethod(Protocol):\n" + " def method(self) -> int: ...\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(x: HasMethod):\n" + " return x.method()\n"); + ASSERT_NOT_NULL(r); + /* Should resolve x.method against the Protocol */ + int idx = find_resolved(r, "use", "method"); + if (idx < 0) printf(" KNOWN GAP: Protocol structural method dispatch\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_abc_abstractmethod) { + /* class Base(ABC): @abstractmethod def m(self): ... + * class Child(Base): def m(self): return 1 + * def use(x: Base): x.m() → resolves to Base.m (or Child.m via dispatch?) */ + CBMFileResult *r = extract_py( + "from abc import ABC, abstractmethod\n" + "class Base(ABC):\n" + " @abstractmethod\n" + " def method(self):\n" + " ...\n" + "class Child(Base):\n" + " def method(self):\n" + " return 1\n" + "def use(x: Base):\n" + " return x.method()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +/* ── Control flow / binding ─────────────────────────────────── */ + +TEST(stress_context_manager_with_as) { + /* with Foo() as f: f.method() — f bound by __enter__ */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def __enter__(self):\n" + " return self\n" + " def __exit__(self, *args):\n" + " return None\n" + " def method(self):\n" + " return 1\n" + "def use():\n" + " with Foo() as f:\n" + " return f.method()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_except_as_binding) { + /* except E as e: e.method() — e bound to E */ + CBMFileResult *r = extract_py( + "class FooError(Exception):\n" + " def detail(self):\n" + " return 'oops'\n" + "def use():\n" + " try:\n" + " return 1\n" + " except FooError as e:\n" + " return e.detail()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "detail"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_isinstance_else_negative_narrow) { + /* if not isinstance(x, Foo): return; x.method() + * Pyright narrows x to Foo after the early return. We don't model this. */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(x):\n" + " if not isinstance(x, Foo):\n" + " return None\n" + " return x.method()\n"); + ASSERT_NOT_NULL(r); + int idx = find_resolved(r, "use", "method"); + if (idx < 0) printf(" KNOWN GAP: post-early-return narrowing\n"); + cbm_free_result(r); + PASS(); +} + +/* ── Containers / unpacking ─────────────────────────────────── */ + +TEST(stress_tuple_unpack_function_return) { + /* def f() -> tuple[Foo, Bar]: ... + * a, b = f(); a.method_a(); b.method_b() */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method_a(self):\n" + " return 1\n" + "class Bar:\n" + " def method_b(self):\n" + " return 2\n" + "def make() -> tuple[Foo, Bar]:\n" + " return Foo(), Bar()\n" + "def use():\n" + " a, b = make()\n" + " a.method_a()\n" + " b.method_b()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method_a"), 0); + ASSERT_GTE(require_resolved(r, "use", "method_b"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_dict_items_comprehension) { + /* d: dict[str, Foo]; [v.method() for k, v in d.items()] */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(d: dict[str, Foo]):\n" + " return [v.method() for k, v in d.items()]\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_list_slice_returns_list) { + /* lst: list[Foo]; lst[1:3][0].method() */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(items: list[Foo]):\n" + " return items[1:3][0].method()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +/* ── Decorators / first-class functions ──────────────────────── */ + +TEST(stress_function_as_dict_value) { + /* funcs: dict[str, Callable] = {"a": foo}; funcs["a"]() */ + CBMFileResult *r = extract_py( + "def foo():\n" + " return 1\n" + "def bar():\n" + " return 2\n" + "def use():\n" + " funcs = {'a': foo, 'b': bar}\n" + " funcs['a']()\n"); + ASSERT_NOT_NULL(r); + /* funcs['a'] resolves to a Callable; calling it is an unknown target. + * This is genuinely hard without runtime info. */ + int idx = find_resolved(r, "use", "foo"); + if (idx < 0) printf(" KNOWN GAP: function-as-dict-value indirect call\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_decorator_factory) { + /* @retry(times=3) — decorator factory pattern */ + CBMFileResult *r = extract_py( + "import functools\n" + "def retry(times: int):\n" + " def deco(fn):\n" + " @functools.wraps(fn)\n" + " def wrapper(*args, **kwargs):\n" + " return fn(*args, **kwargs)\n" + " return wrapper\n" + " return deco\n" + "@retry(times=3)\n" + "def helper():\n" + " return 1\n" + "def use():\n" + " return helper()\n"); + ASSERT_NOT_NULL(r); + /* helper() should resolve as helper despite the decorator. */ + ASSERT_GTE(require_resolved(r, "use", "helper"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_property_setter) { + /* @prop.setter — assignment to property */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " @property\n" + " def value(self) -> int:\n" + " return 1\n" + " @value.setter\n" + " def value(self, v: int) -> None:\n" + " pass\n" + " def use(self):\n" + " self.value = 5\n" + " return self.value\n"); + ASSERT_NOT_NULL(r); + /* Setter is rare in resolution but should not break anything. */ + cbm_free_result(r); + PASS(); +} + +/* ── Inheritance / generics ─────────────────────────────────── */ + +TEST(stress_self_in_inheritance_chain) { + /* class B: def make(self) -> Self: ... + * class C(B): def child_method(self) -> int: ... + * c = C().make() # Pyright: c is C, not B + * c.child_method() */ + CBMFileResult *r = extract_py( + "from typing import Self\n" + "class Base:\n" + " def make(self) -> Self:\n" + " return self\n" + "class Child(Base):\n" + " def child_method(self):\n" + " return 1\n" + "def use():\n" + " c = Child().make()\n" + " return c.child_method()\n"); + ASSERT_NOT_NULL(r); + int idx = find_resolved(r, "use", "child_method"); + if (idx < 0) printf(" KNOWN GAP: Self resolves to receiver class, not declaring class\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_diamond_inheritance_mro) { + /* Diamond: Top -> Left, Right -> Bottom. Bottom().method() resolves to + * Top.method via C3 unless Left/Right override. */ + CBMFileResult *r = extract_py( + "class Top:\n" + " def method(self):\n" + " return 1\n" + "class Left(Top):\n" + " pass\n" + "class Right(Top):\n" + " pass\n" + "class Bottom(Left, Right):\n" + " def use(self):\n" + " return self.method()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_recursive_self_referencing_type) { + /* class Node: children: list[Node] + * def walk(node: Node): for c in node.children: c.method() */ + CBMFileResult *r = extract_py( + "class Node:\n" + " def __init__(self):\n" + " self.children: list['Node'] = []\n" + " def method(self):\n" + " return 1\n" + "def walk(node: Node):\n" + " for c in node.children:\n" + " c.method()\n"); + ASSERT_NOT_NULL(r); + int idx = find_resolved(r, "walk", "method"); + if (idx < 0) printf(" KNOWN GAP: forward-ref `Node` inside list[] within own class\n"); + cbm_free_result(r); + PASS(); +} + +/* ── Advanced control flow ─────────────────────────────────── */ + +TEST(stress_nested_closure) { + /* def outer(): + * x = Foo() + * def inner(): + * return x.method() + * return inner */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def outer():\n" + " x = Foo()\n" + " def inner():\n" + " return x.method()\n" + " return inner()\n"); + ASSERT_NOT_NULL(r); + /* inner() captures x via closure; x is Foo from outer scope. */ + int idx = find_resolved(r, "inner", "method"); + if (idx < 0) printf(" KNOWN GAP: closure scope capture across nested function\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_match_sequence_pattern) { + /* match items: case [head, *tail]: head.method() */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(items: list[Foo]):\n" + " match items:\n" + " case [head, *tail]:\n" + " return head.method()\n" + " case []:\n" + " return None\n"); + ASSERT_NOT_NULL(r); + int idx = find_resolved(r, "use", "method"); + if (idx < 0) printf(" KNOWN GAP: match sequence pattern element typing\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_lambda_inference) { + /* fn = lambda x: x.method(); fn(Foo()) */ + CBMFileResult *r = extract_py( + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use():\n" + " fn = lambda x: x.method()\n" + " return fn(Foo())\n"); + ASSERT_NOT_NULL(r); + /* x's type inside the lambda comes from the call site — needs + * bidirectional inference. We don't model this. */ + int idx = find_resolved(r, "", "method"); + if (idx < 0) printf(" KNOWN GAP: lambda parameter inference from call site\n"); + cbm_free_result(r); + PASS(); +} + +TEST(stress_method_chain_long) { + /* a.b().c().d().e() — every step must preserve type for the next. */ + CBMFileResult *r = extract_py( + "from typing import Self\n" + "class B:\n" + " def step(self) -> Self:\n" + " return self\n" + " def finish(self):\n" + " return 1\n" + "def use():\n" + " return B().step().step().step().step().finish()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "finish"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_generator_delegation) { + /* def outer(): yield from inner() — outer's iterable element is inner's */ + CBMFileResult *r = extract_py( + "from typing import Generator\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def inner() -> Generator[Foo, None, None]:\n" + " yield Foo()\n" + "def outer() -> Generator[Foo, None, None]:\n" + " yield from inner()\n" + "def use():\n" + " for x in outer():\n" + " x.method()\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "use", "method"), 0); + cbm_free_result(r); + PASS(); +} + +TEST(stress_async_gen_for) { + /* async for x in async_gen(): x.method() */ + CBMFileResult *r = extract_py( + "from typing import AsyncGenerator\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "async def gen() -> AsyncGenerator[Foo, None]:\n" + " yield Foo()\n" + "async def use():\n" + " async for x in gen():\n" + " x.method()\n"); + ASSERT_NOT_NULL(r); + int idx = find_resolved(r, "use", "method"); + if (idx < 0) printf(" KNOWN GAP: async-for element typing\n"); + cbm_free_result(r); + PASS(); +} + +/* ── Suite ─────────────────────────────────────────────────── */ + +SUITE(py_lsp_stress) { + /* Typing helpers */ + RUN_TEST(stress_namedtuple_class_form); + RUN_TEST(stress_typeddict_subscript); + RUN_TEST(stress_protocol_structural); + RUN_TEST(stress_abc_abstractmethod); + /* Control flow / binding */ + RUN_TEST(stress_context_manager_with_as); + RUN_TEST(stress_except_as_binding); + RUN_TEST(stress_isinstance_else_negative_narrow); + /* Containers */ + RUN_TEST(stress_tuple_unpack_function_return); + RUN_TEST(stress_dict_items_comprehension); + RUN_TEST(stress_list_slice_returns_list); + /* Decorators / first-class functions */ + RUN_TEST(stress_function_as_dict_value); + RUN_TEST(stress_decorator_factory); + RUN_TEST(stress_property_setter); + /* Inheritance / generics */ + RUN_TEST(stress_self_in_inheritance_chain); + RUN_TEST(stress_diamond_inheritance_mro); + RUN_TEST(stress_recursive_self_referencing_type); + /* Advanced control flow */ + RUN_TEST(stress_nested_closure); + RUN_TEST(stress_match_sequence_pattern); + RUN_TEST(stress_lambda_inference); + RUN_TEST(stress_method_chain_long); + RUN_TEST(stress_generator_delegation); + RUN_TEST(stress_async_gen_for); +}