diff --git a/Makefile.cbm b/Makefile.cbm index ad74c81c5..23f4f2985 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -612,7 +612,8 @@ PIPELINE_SRCS = \ src/pipeline/pass_cross_repo.c \ src/pipeline/artifact.c \ src/pipeline/pass_ensemble_routing.c \ - src/pipeline/pass_pkgmap.c + src/pipeline/pass_pkgmap.c \ + src/pipeline/py_layout.c # SimHash / MinHash module SIMHASH_SRCS = src/simhash/minhash.c diff --git a/internal/cbm/resolve/binding.c b/internal/cbm/resolve/binding.c index e25d02550..06b45d8b6 100644 --- a/internal/cbm/resolve/binding.c +++ b/internal/cbm/resolve/binding.c @@ -69,6 +69,9 @@ typedef struct { const char *value_chain; uint32_t value_scope; bool value_none; + /* Python: from value_from to value_until every path reaches this binder. Zero until: + * no such stretch. */ + uint32_t value_from, value_until; bool import_root; } binding_name_t; @@ -165,6 +168,35 @@ typedef struct { const cbm_resolve_proof_t *proof; } binding_observation_t; +typedef struct { + uint32_t start, end; + uint32_t parent; /* one-based enclosing loop */ +} python_loop_t; + +/* `if isinstance(E, C):` and its kin: in [from, until] E holds a C. */ +typedef struct { + const char *chain, *type; + uint32_t from, until, scope; + uint32_t parent; /* one-based enclosing narrowing */ +} python_narrow_t; + +typedef struct { + const char *chain; + uint32_t at; /* where the statement ends: its right side still reads what was there */ + uint32_t scope; + const char *value; /* what a plain assignment stores, when followable */ + uint32_t value_at; +} python_store_t; + +typedef struct { + const char *name; /* the local name the statement went through */ + const char *member; /* the dotted path it writes, NULL when any name can change */ +} python_write_t; + +typedef struct { + const char *alias, *of; +} python_alias_t; + struct cbm_resolve_binding_index { CBMArena *arena; binding_observation_t *observation; @@ -183,9 +215,21 @@ struct cbm_resolve_binding_index { bool javascript_module; bool python_annotation_binding_preserved; bool python_annotation_storage_bound; - const char **python_escapes; - size_t python_escape_count, python_escape_capacity; + python_write_t *python_writes; /* sorted by name */ + python_alias_t *python_aliases; /* `alias = name` statements */ + size_t python_alias_count, python_alias_capacity; + size_t python_write_count, python_write_capacity; bool python_effects_complete; + bool python_writes_unseen; /* the audit stopped before the end of the file */ + bool python_name_handed; /* __name__ goes to a call: that may build the module */ + bool python_namespace_open; /* a computed globals() store can add any name */ + python_narrow_t *python_narrows; /* sorted by from, outermost first, once the walk ends */ + size_t python_narrow_count, python_narrow_capacity; + python_store_t *python_stores; /* attribute stores, sorted by chain then position */ + size_t python_store_count, python_store_capacity; + python_loop_t *python_loops; /* in start order: the walk's */ + size_t python_loop_count, python_loop_capacity; + uint32_t python_open_loop; /* one-based innermost loop being walked */ cbm_resolve_python_nested_import_t *python_nested; size_t python_nested_count, python_nested_capacity; cbm_resolve_python_star_t *python_stars; @@ -196,9 +240,6 @@ struct cbm_resolve_binding_index { size_t python_member_count, python_member_capacity; cbm_resolve_python_decorated_t *python_decorated; size_t python_decorated_count, python_decorated_capacity; - /* Set while python_decorator_shape reads one function's returns. */ - const char *python_shape_delegate; - bool python_shape_delegate_applied; const char *http_default_target, *http_default_base; cbm_resolve_http_dependency_t *http_dependencies; size_t http_dependency_count, http_dependency_capacity; @@ -415,6 +456,51 @@ static bool jsx_component_site(const cbm_resolve_binding_index_t *index, TSNode index->source[start] <= 'Z'; } +static const char *python_import_canonical(const cbm_resolve_binding_index_t *index, + const char *chain); +static const char *python_annotation_chain(cbm_resolve_binding_index_t *index, TSNode node, + unsigned depth); + +const char *cbm_resolve_python_typed_text(CBMArena *arena, const char *type, const char *rest) { + const char *marker = strchr(type, '@'); + size_t head = marker ? (size_t)(marker - type) : strlen(type); + if (marker && !strcmp(marker, "@type")) + return cbm_arena_sprintf(arena, "%.*s%s", (int)head, type, rest); + return cbm_arena_sprintf(arena, "%.*s()%s%s", (int)head, type, marker ? marker : "", rest); +} + +static bool listed(const char *kind, const char *const *list); + +/* The enclosing def declares this name as a parameter whose default is None. */ +static bool python_parameter_defaults_none(cbm_resolve_binding_index_t *index, TSNode node, + const char *name) { + TSNode function = node; + for (unsigned depth = 0; depth < BINDING_MAX_DEPTH && !ts_node_is_null(function); depth++) { + if (kind_is(function, "lambda") || kind_is(function, "class_definition")) + return false; + if (kind_is(function, "function_definition")) + break; + function = ts_node_parent(function); + } + TSNode parameters = + kind_is(function, "function_definition") ? field(function, "parameters") : (TSNode){0}; + if (ts_node_is_null(parameters) || !name) + return false; + bool found = false; + TSTreeCursor cursor = ts_tree_cursor_new(parameters); + for (bool more = ts_tree_cursor_goto_first_child(&cursor); more && !found; + more = ts_tree_cursor_goto_next_sibling(&cursor)) { + TSNode parameter = ts_tree_cursor_current_node(&cursor); + if (!kind_is(parameter, "default_parameter") && + !kind_is(parameter, "typed_default_parameter")) + continue; + const char *spelled = node_text(index, field(parameter, "name")); + found = spelled && !strcmp(spelled, name) && kind_is(field(parameter, "value"), "none"); + } + ts_tree_cursor_delete(&cursor); + return found; +} + /* The access path of a Python expression, as text: names, member steps and * calls with their arguments dropped. Only forms whose value can be followed * statically have one. Two-argument super and one-argument type keep their @@ -435,6 +521,30 @@ static const char *python_chain(cbm_resolve_binding_index_t *index, TSNode node, const char *member = node_text(index, field(node, "attribute")); return object && member ? cbm_arena_sprintf(index->arena, "%s.%s", object, member) : NULL; } + /* x[k] and x[a:b]: what the container holds, read at evaluation. */ + if (kind_is(node, "subscript") && ts_node_named_child_count(node) == 2) { + const char *object = python_chain(index, field(node, "value"), depth + 1); + return object ? cbm_arena_sprintf(index->arena, "%s%s", object, + kind_is(field(node, "subscript"), "slice") ? "[:]" : "[]") + : NULL; + } + /* `x or Y()` for a parameter defaulting to None: Y() is what the signature provides. */ + if (kind_is(node, "boolean_operator")) { + const char *operator= node_text(index, field(node, "operator")); + TSNode left = field(node, "left"); + + if (operator&& !strcmp(operator, "or") && kind_is(left, "identifier") && + python_parameter_defaults_none(index, node, node_text(index, left))) + return python_chain(index, field(node, "right"), depth + 1); + return NULL; + } + /* `A() if c else None`: a None says nothing about the class a member read finds. */ + if (kind_is(node, "conditional_expression") && ts_node_named_child_count(node) == 3) { + TSNode first = ts_node_named_child(node, 0), second = ts_node_named_child(node, 2); + if (kind_is(first, "none") != kind_is(second, "none")) + return python_chain(index, kind_is(first, "none") ? second : first, depth + 1); + return NULL; + } if (!kind_is(node, "call")) return NULL; TSNode function = field(node, "function"), args = field(node, "arguments"); @@ -448,6 +558,30 @@ static const char *python_chain(cbm_resolve_binding_index_t *index, TSNode node, if (wrapped) return cbm_arena_sprintf(index->arena, "%s(%s)", callee, wrapped); } + /* copy.copy(x) is an x, by the standard library's own signature. */ + size_t spelled = strlen(callee); + if (count == 1 && ((spelled >= 4 && !strcmp(callee + spelled - 4, "copy")))) { + const char *canonical = python_import_canonical(index, callee); + if (canonical && (!strcmp(canonical, "copy.copy") || !strcmp(canonical, "copy.deepcopy"))) + return python_chain(index, ts_node_named_child(args, 0), depth + 1); + } + /* typing.cast(T, x) is a T: that is all it does. */ + if (count == 2 && spelled >= 4 && !strcmp(callee + spelled - 4, "cast")) { + const char *canonical = python_import_canonical(index, callee); + if (canonical && + (!strcmp(canonical, "typing.cast") || !strcmp(canonical, "typing_extensions.cast"))) { + const char *type = python_annotation_chain(index, ts_node_named_child(args, 0), 0); + return type ? cbm_resolve_python_typed_text(index->arena, type, "") : NULL; + } + } + /* sorted(x) and friends hold what x holds; py_members checks the name is the builtin. */ + static const char *const passing[] = {"list", "tuple", "set", "frozenset", + "sorted", "reversed", "iter", NULL}; + if (count >= 1 && listed(callee, passing)) { + const char *wrapped = python_chain(index, ts_node_named_child(args, 0), depth + 1); + if (wrapped) + return cbm_arena_sprintf(index->arena, "%s(%s)", callee, wrapped); + } bool owner_form = (!strcmp(callee, "super") && count == 2) || (!strcmp(callee, "type") && count == 1); if (!owner_form) @@ -872,10 +1006,26 @@ static bool python_namespace_call_writes(cbm_resolve_binding_index_t *index, TSN TSNode parent = ts_node_parent(call); if (kind_is(parent, "comparison_operator") || kind_is(parent, "dictionary_splat")) return false; + /* exec(code, globals(), d) stores in d. */ + if (kind_is(parent, "argument_list") && ts_node_named_child_count(parent) == 3 && + ts_node_eq(call, ts_node_named_child(parent, 1))) { + const char *runner = node_text(index, field(ts_node_parent(parent), "function")); + if (runner && (!strcmp(runner, "exec") || !strcmp(runner, "eval"))) + return false; + } if (kind_is(parent, "subscript")) { - TSNode user = ts_node_parent(parent); - return (kind_is(user, "assignment") || kind_is(user, "augmented_assignment")) && - ts_node_eq(parent, field(user, "left")); + TSNode user = ts_node_parent(parent), key = field(parent, "subscript"); + bool stored = (kind_is(user, "assignment") || kind_is(user, "augmented_assignment")) && + ts_node_eq(parent, field(user, "left")); + bool literal = kind_is(key, "string") && ts_node_named_child_count(key) == 3 && + kind_is(ts_node_named_child(key, 1), "string_content"); + /* A computed key names no binding this audit saw. Names bound by statements stay what + * those say, as celery's STATICA_HACK relies on; what the module lacks is unknowable. */ + if (stored && !literal) { + index->python_namespace_open = true; + return false; + } + return stored; } if (kind_is(parent, "attribute")) { const char *member = node_text(index, field(parent, "attribute")); @@ -890,13 +1040,22 @@ static bool python_namespace_call_writes(cbm_resolve_binding_index_t *index, TSN * A function's locals() is a snapshot: writing to it rebinds nothing, and vars(x) * reads x. Only the zero-argument form outside a function can write a binding; * any other read of those names passes the view on and stays unknown. */ -static bool python_namespace_view(const cbm_resolve_binding_index_t *index, TSNode node, +static bool python_namespace_view(cbm_resolve_binding_index_t *index, TSNode node, const char *name) { if (strcmp(name, "locals") && strcmp(name, "vars")) return false; TSNode call = ts_node_parent(node); if (!kind_is(call, "call") || !ts_node_eq(node, field(call, "function"))) return true; + /* A read-only method of the view leaves the namespace as it was. */ + TSNode user = ts_node_parent(call); + if (kind_is(user, "attribute") && ts_node_eq(call, field(user, "object"))) { + const char *member = node_text(index, field(user, "attribute")); + if (member && + (!strcmp(member, "get") || !strcmp(member, "keys") || !strcmp(member, "items") || + !strcmp(member, "values") || !strcmp(member, "copy"))) + return false; + } bool in_function = index->current_scope && index->current_scope < index->scope_count && !index->scopes[index->current_scope].class_scope; TSNode args = field(call, "arguments"); @@ -994,11 +1153,14 @@ static void note_import(cbm_resolve_binding_index_t *index, TSNode node, bool ne /* Only annotation lookups see these names: runtime reads find a blocked binder. */ static void python_note_annotation_imports(cbm_resolve_binding_index_t *index, TSNode node) { size_t names = index->name_count, nested = index->python_nested_count; + size_t stars = index->python_star_count; note_import(index, node, true); for (size_t i = names; i < index->name_count; i++) index->names[i].flags = BINDING_BLOCKED; for (size_t i = nested; i < index->python_nested_count; i++) index->python_nested[i].annotation_only = true; + for (size_t i = stars; i < index->python_star_count; i++) + index->python_stars[i].annotation_only = true; } static void note_import(cbm_resolve_binding_index_t *index, TSNode node, bool nested) { @@ -1166,14 +1328,6 @@ static const char *python_import_canonical(const cbm_resolve_binding_index_t *in static const char *python_annotation_chain(cbm_resolve_binding_index_t *index, TSNode node, unsigned depth); -/* typing.Optional however it was imported. A project class of that name says nothing - * about its argument. */ -static bool python_is_typing_optional(cbm_resolve_binding_index_t *index, TSNode outer) { - const char *spelled = python_chain(index, outer, 0); - const char *canonical = spelled ? python_import_canonical(index, spelled) : NULL; - return canonical && !strcmp(canonical, "typing.Optional"); -} - static bool python_annotation_is_none(TSNode node) { while (kind_is(node, "type") && ts_node_named_child_count(node) == 1) node = ts_node_named_child(node, 0); @@ -1181,6 +1335,98 @@ static bool python_annotation_is_none(TSNode node) { } /* `X | None` says X. Two classes say neither. */ +/* What a generic annotation says a value is. Its marker: @seq holds and yields X, @map holds + * X by key, @type is the class X itself. A generic of the project's own is an instance of it. */ +static const char *python_generic_chain(cbm_resolve_binding_index_t *index, TSNode head, + const TSNode *args, uint32_t count, unsigned depth) { + static const char *const seq[] = {"list", + "set", + "frozenset", + "typing.List", + "typing.Set", + "typing.FrozenSet", + "typing.Sequence", + "typing.MutableSequence", + "typing.Iterable", + "typing.Iterator", + "typing.Collection", + "typing.Generator", + "typing.AsyncIterator", + "typing.AsyncIterable", + "typing.AsyncGenerator", + "typing.Deque", + "typing.Reversible", + "typing.AbstractSet", + "typing.MutableSet", + "collections.abc.Sequence", + "collections.abc.MutableSequence", + "collections.abc.Iterable", + "collections.abc.Iterator", + "collections.abc.Collection", + "collections.abc.Generator", + "collections.abc.AsyncIterator", + "collections.abc.AsyncIterable", + "collections.abc.AsyncGenerator", + "collections.abc.Set", + "collections.abc.MutableSet", + "collections.deque", + NULL}; + static const char *const map[] = {"dict", + "typing.Dict", + "typing.Mapping", + "typing.MutableMapping", + "typing.DefaultDict", + "typing.OrderedDict", + "collections.abc.Mapping", + "collections.abc.MutableMapping", + "collections.OrderedDict", + "collections.defaultdict", + NULL}; + static const char *const wrapper[] = { + "typing.Optional", "typing.Annotated", + "typing.ClassVar", "typing.Final", + "typing.Awaitable", "typing_extensions.Annotated", + "collections.abc.Awaitable", NULL}; + const char *spelled = python_chain(index, head, 0); + if (!spelled || !count) + return NULL; + const char *canonical = python_import_canonical(index, spelled); + if (!canonical) + canonical = spelled; + const char *marker = NULL; + uint32_t arg = 0; + if (listed(canonical, wrapper)) { + return python_annotation_chain(index, args[0], depth + 1); + } else if (!strcmp(canonical, "typing.Coroutine") || + !strcmp(canonical, "collections.abc.Coroutine")) { + return count == 3 ? python_annotation_chain(index, args[2], depth + 1) : NULL; + } else if (listed(canonical, seq)) { + marker = "@seq"; + } else if (!strcmp(canonical, "tuple") || !strcmp(canonical, "typing.Tuple")) { + if (count != 2 || !kind_is(args[1], "type") || !ts_node_named_child_count(args[1]) || + !kind_is(ts_node_named_child(args[1], 0), "ellipsis")) + return NULL; + marker = "@seq"; + } else if (listed(canonical, map)) { + if (count != 2) + return NULL; + marker = "@map"; + arg = 1; + } else if (!strcmp(canonical, "type") || !strcmp(canonical, "typing.Type")) { + marker = "@type"; + } else if (!strncmp(canonical, "typing.", 7) || !strncmp(canonical, "typing_extensions.", 18) || + !strncmp(canonical, "collections.", 12)) { + return NULL; + } else { + return spelled; /* Repo[str] is a Repo. */ + } + const char *element = python_annotation_chain(index, args[arg], depth + 1); + /* One layer: list[list[X]] says nothing this model can carry. */ + return element && !strchr(element, '@') + ? cbm_arena_sprintf(index->arena, "%s%s", element, marker) + : NULL; +} + static const char *python_union_chain(cbm_resolve_binding_index_t *index, TSNode left, TSNode right, unsigned depth) { bool left_none = python_annotation_is_none(left), right_none = python_annotation_is_none(right); @@ -1209,17 +1455,26 @@ static const char *python_annotation_chain(cbm_resolve_binding_index_t *index, T return cbm_arena_strndup(index->arena, text + 1, length - 2); } if (kind_is(node, "generic_type") && ts_node_named_child_count(node) == 2) { - TSNode arguments = ts_node_named_child(node, 1); - if (!python_is_typing_optional(index, ts_node_named_child(node, 0)) || - ts_node_named_child_count(arguments) != 1) - return NULL; - return python_annotation_chain(index, ts_node_named_child(arguments, 0), depth + 1); + TSNode arguments = ts_node_named_child(node, 1), args[3]; + memset(args, 0, sizeof(args)); + uint32_t count = ts_node_named_child_count(arguments); + for (uint32_t i = 0; i < count && i < 3; i++) + args[i] = ts_node_named_child(arguments, i); + return count <= 3 + ? python_generic_chain(index, ts_node_named_child(node, 0), args, count, depth) + : NULL; } /* The grammar reads a dotted generic, typing.Optional[X], as a subscript. */ - if (kind_is(node, "subscript") && ts_node_named_child_count(node) == 2) - return python_is_typing_optional(index, field(node, "value")) - ? python_annotation_chain(index, field(node, "subscript"), depth + 1) + if (kind_is(node, "subscript")) { + TSNode args[3]; + memset(args, 0, sizeof(args)); + uint32_t count = ts_node_named_child_count(node), used = 0; + for (uint32_t i = 1; i < count && used < 3; i++) + args[used++] = ts_node_named_child(node, i); + return count >= 2 && count <= 4 + ? python_generic_chain(index, field(node, "value"), args, used, depth) : NULL; + } if (kind_is(node, "union_type") && ts_node_named_child_count(node) == 2) return python_union_chain(index, ts_node_named_child(node, 0), ts_node_named_child(node, 1), depth); @@ -1251,12 +1506,6 @@ static const char *python_import_canonical(const cbm_resolve_binding_index_t *in return NULL; } -static TSNode python_function_of(TSNode statement) { - if (kind_is(statement, "decorated_definition")) - statement = field(statement, "definition"); - return kind_is(statement, "function_definition") ? statement : (TSNode){0}; -} - /* Every path through the block ends in return or raise: it cannot fall off the * end, which would rebind the decorated name to None. */ static bool python_block_always_exits(TSNode block, unsigned depth) { @@ -1291,283 +1540,6 @@ static bool python_block_always_exits(TSNode block, unsigned depth) { return has_else; } -static bool python_calls_name(cbm_resolve_binding_index_t *index, TSNode node, const char *name, - unsigned depth) { - if (ts_node_is_null(node) || depth > 48) - return false; - if (kind_is(node, "call")) { - TSNode function = field(node, "function"); - const char *text = kind_is(function, "identifier") ? node_text(index, function) : NULL; - if (text && !strcmp(text, name)) - return true; - } - if (depth && (kind_is(node, "class_definition") || kind_is(node, "function_definition"))) - return false; - for (uint32_t i = 0; i < ts_node_named_child_count(node); i++) - if (python_calls_name(index, ts_node_named_child(node, i), name, depth + 1)) - return true; - return false; -} - -/* `@wraps(fn)` on a local def whose body calls fn: calling the wrapper is calling - * the function it wraps. */ -static bool python_wrapper_of(cbm_resolve_binding_index_t *index, TSNode function, const char *fn) { - TSNode decorated = ts_node_parent(function); - if (!kind_is(decorated, "decorated_definition")) - return false; - for (uint32_t i = 0; i < ts_node_named_child_count(decorated); i++) { - TSNode decorator = ts_node_named_child(decorated, i); - if (!kind_is(decorator, "decorator") || !ts_node_named_child_count(decorator)) - continue; - TSNode call = ts_node_named_child(decorator, 0); - TSNode args = kind_is(call, "call") ? field(call, "arguments") : (TSNode){0}; - if (ts_node_is_null(args) || !ts_node_named_child_count(args)) - continue; - const char *spelled = python_chain(index, field(call, "function"), 0); - const char *canonical = spelled ? python_import_canonical(index, spelled) : NULL; - const char *first = kind_is(ts_node_named_child(args, 0), "identifier") - ? node_text(index, ts_node_named_child(args, 0)) - : NULL; - if (canonical && !strcmp(canonical, "functools.wraps") && first && !strcmp(first, fn)) - return python_calls_name(index, field(function, "body"), fn, 0); - } - return false; -} - -static void python_collect_returns(cbm_resolve_binding_index_t *index, TSNode node, TSNode owner, - const char *parameter, unsigned depth, unsigned *shape, - bool *failed); - -/* What applying this function to a decorated declaration does, from its source. - * DIRECT: `@deco` keeps the declaration callable as itself. FACTORY: `@deco(...)` - * does. Every return must be accounted for; one unknown return proves nothing. */ -static unsigned python_decorator_shape(cbm_resolve_binding_index_t *index, TSNode function, - unsigned depth) { - TSNode body = field(function, "body"), parameters = field(function, "parameters"); - if (depth > 4 || ts_node_is_null(body) || ts_node_is_null(parameters) || - !python_block_always_exits(body, 0)) - return 0; - TSNode first = - ts_node_named_child_count(parameters) ? ts_node_named_child(parameters, 0) : (TSNode){0}; - if (kind_is(first, "typed_parameter") && ts_node_named_child_count(first)) - first = ts_node_named_child(first, 0); - if (kind_is(first, "default_parameter") || kind_is(first, "typed_default_parameter")) - first = field(first, "name"); - const char *parameter = kind_is(first, "identifier") ? node_text(index, first) : NULL; - unsigned shape = 0; - bool failed = false; - if (depth == 0) { - index->python_shape_delegate = NULL; - index->python_shape_delegate_applied = false; - } - python_collect_returns(index, body, function, parameter, depth, &shape, &failed); - return failed ? 0 : shape; -} - -typedef bool (*python_def_check_t)(cbm_resolve_binding_index_t *, TSNode, const char *, unsigned); - -static bool python_def_calls(cbm_resolve_binding_index_t *index, TSNode function, const char *fn, - unsigned depth) { - (void)depth; - return python_calls_name(index, field(function, "body"), fn, 0); -} - -static bool python_def_wraps(cbm_resolve_binding_index_t *index, TSNode function, const char *fn, - unsigned depth) { - (void)depth; - return python_wrapper_of(index, function, fn); -} - -static bool python_def_decorates(cbm_resolve_binding_index_t *index, TSNode function, - const char *fn, unsigned depth) { - (void)fn; - return python_decorator_shape(index, function, depth) & CBM_PY_SHAPE_DIRECT; -} - -static bool python_walk_local_defs(cbm_resolve_binding_index_t *index, TSNode node, - const char *name, const char *fn, python_def_check_t check, - unsigned check_depth, unsigned *found, unsigned depth) { - if (ts_node_is_null(node) || depth > 24) - return false; - for (uint32_t i = 0; i < ts_node_named_child_count(node); i++) { - TSNode child = ts_node_named_child(node, i); - TSNode function = python_function_of(child); - if (!ts_node_is_null(function)) { - const char *text = node_text(index, field(function, "name")); - if (text && !strcmp(text, name)) { - (*found)++; - if (!check(index, function, fn, check_depth)) - return false; - } - continue; - } - if (kind_is(child, "class_definition") || kind_is(child, "lambda") || - kind_is(child, "expression_statement") || kind_is(child, "return_statement")) - continue; - if (!python_walk_local_defs(index, child, name, fn, check, check_depth, found, depth + 1)) - return false; - } - return true; -} - -/* The defs of `name` in this function's own body, outside nested scopes. Either arm - * of a conditional pair can be the one bound at run time, so every def has to pass. */ -static bool python_local_defs_pass(cbm_resolve_binding_index_t *index, TSNode body, - const char *name, const char *fn, python_def_check_t check, - unsigned check_depth) { - unsigned found = 0; - return name && python_walk_local_defs(index, body, name, fn, check, check_depth, &found, 0) && - found; -} - -static bool python_is_parameter(cbm_resolve_binding_index_t *index, TSNode node, - const char *parameter) { - const char *text = parameter && kind_is(node, "identifier") ? node_text(index, node) : NULL; - return text && !strcmp(text, parameter); -} - -static TSNode python_call_argument(TSNode call, uint32_t position) { - TSNode args = field(call, "arguments"); - return !ts_node_is_null(args) && ts_node_named_child_count(args) > position - ? ts_node_named_child(args, position) - : (TSNode){0}; -} - -/* update_wrapper(g, fn) anywhere in this body, outside nested scopes. */ -static bool python_body_updates_wrapper(cbm_resolve_binding_index_t *index, TSNode node, - const char *wrapper, const char *parameter, - unsigned depth) { - if (ts_node_is_null(node) || depth > 16) - return false; - for (uint32_t i = 0; i < ts_node_named_child_count(node); i++) { - TSNode child = ts_node_named_child(node, i); - if (kind_is(child, "function_definition") || kind_is(child, "class_definition") || - kind_is(child, "decorated_definition") || kind_is(child, "lambda")) - continue; - if (kind_is(child, "call")) { - const char *callee = python_chain(index, field(child, "function"), 0); - const char *imported = callee ? python_import_canonical(index, callee) : NULL; - if (imported && !strcmp(imported, "functools.update_wrapper") && - python_is_parameter(index, python_call_argument(child, 0), wrapper) && - python_is_parameter(index, python_call_argument(child, 1), parameter)) - return true; - } - if (python_body_updates_wrapper(index, child, wrapper, parameter, depth + 1)) - return true; - } - return false; -} - -/* The return hands the function to another decorator: X(fn) or X(...)(fn). Which - * function X names, and what it does, is decided across files. */ -static bool python_note_delegate(cbm_resolve_binding_index_t *index, TSNode callee, bool applied, - unsigned *shape) { - const char *chain = python_chain(index, callee, 0); - if (!chain || (index->python_shape_delegate && strcmp(index->python_shape_delegate, chain))) - return false; - index->python_shape_delegate = chain; - index->python_shape_delegate_applied = applied; - *shape |= CBM_PY_SHAPE_DELEGATES; - return true; -} - -static void python_classify_call_return(cbm_resolve_binding_index_t *index, TSNode value, - TSNode owner, const char *parameter, unsigned depth, - unsigned *shape, bool *failed) { - TSNode body = field(owner, "body"), callee = field(value, "function"); - const char *name = kind_is(callee, "identifier") ? node_text(index, callee) : NULL; - /* decorator(func), the both-ways form: a local decorator applied here. */ - if (python_local_defs_pass(index, body, name, NULL, python_def_decorates, depth + 1)) { - *shape |= CBM_PY_SHAPE_DIRECT; - return; - } - TSNode first = python_call_argument(value, 0); - const char *first_name = kind_is(first, "identifier") ? node_text(index, first) : NULL; - bool wrapper_calls = parameter && python_local_defs_pass(index, body, first_name, parameter, - python_def_calls, 0); - /* wraps(fn)(g) and update_wrapper(g, fn): g calls fn. */ - if (kind_is(callee, "call")) { - const char *outer = python_chain(index, field(callee, "function"), 0); - const char *imported = outer ? python_import_canonical(index, outer) : NULL; - if (imported && !strcmp(imported, "functools.wraps") && wrapper_calls && - python_is_parameter(index, python_call_argument(callee, 0), parameter)) { - *shape |= CBM_PY_SHAPE_DIRECT; - return; - } - if (depth == 0 && python_is_parameter(index, first, parameter) && - python_note_delegate(index, field(callee, "function"), true, shape)) - return; - *failed = true; - return; - } - const char *spelled = python_chain(index, callee, 0); - const char *imported = spelled ? python_import_canonical(index, spelled) : NULL; - if (imported && !strcmp(imported, "functools.update_wrapper") && wrapper_calls && - python_is_parameter(index, python_call_argument(value, 1), parameter)) { - *shape |= CBM_PY_SHAPE_DIRECT; - return; - } - if (depth == 0 && python_is_parameter(index, first, parameter) && - python_note_delegate(index, callee, false, shape)) - return; - *failed = true; -} - -static void python_classify_return(cbm_resolve_binding_index_t *index, TSNode value, TSNode owner, - const char *parameter, unsigned depth, unsigned *shape, - bool *failed) { - TSNode body = field(owner, "body"); - if (kind_is(value, "identifier")) { - const char *name = node_text(index, value); - if (name && parameter && !strcmp(name, parameter)) { - *shape |= CBM_PY_SHAPE_DIRECT; - return; - } - if (parameter && - (python_local_defs_pass(index, body, name, parameter, python_def_wraps, 0) || - (python_body_updates_wrapper(index, body, name, parameter, 0) && - python_local_defs_pass(index, body, name, parameter, python_def_calls, 0)))) { - *shape |= CBM_PY_SHAPE_DIRECT; - return; - } - if (python_local_defs_pass(index, body, name, NULL, python_def_decorates, depth + 1)) - *shape |= CBM_PY_SHAPE_FACTORY; - else - *failed = true; - return; - } - if (kind_is(value, "call")) { - python_classify_call_return(index, value, owner, parameter, depth, shape, failed); - return; - } - *failed = true; -} - -static void python_collect_returns(cbm_resolve_binding_index_t *index, TSNode node, TSNode owner, - const char *parameter, unsigned depth, unsigned *shape, - bool *failed) { - for (uint32_t i = 0; !*failed && i < ts_node_named_child_count(node); i++) { - TSNode child = ts_node_named_child(node, i); - if (kind_is(child, "function_definition") || kind_is(child, "class_definition") || - kind_is(child, "decorated_definition") || kind_is(child, "lambda")) - continue; - if (kind_is(child, "yield")) { - /* A generator returns a generator, whatever its return says. */ - *failed = true; - return; - } - if (kind_is(child, "return_statement")) { - if (!ts_node_named_child_count(child)) - *failed = true; - else - python_classify_return(index, ts_node_named_child(child, 0), owner, parameter, - depth, shape, failed); - continue; - } - python_collect_returns(index, child, owner, parameter, depth, shape, failed); - } -} - /* Every value this function returns, outside nested scopes. A bare return and * None say nothing about the class of the others. Returns false when one is not * an access path, or the function yields. */ @@ -1606,7 +1578,44 @@ static void python_note_returns(cbm_resolve_binding_index_t *index, TSNode funct enum { RETURNS_MAX = 16 }; const char *chains[RETURNS_MAX]; uint32_t count = 0; - if (member >= index->python_member_count || !ts_node_is_null(field(function, "return_type")) || + /* `-> Self` and `def f(self: T) -> T` return the receiver, whichever class that is. */ + TSNode declared_return = field(function, "return_type"); + binding_scope_t *body_scope = scope < index->scope_count ? &index->scopes[scope] : NULL; + if (member < index->python_member_count && !ts_node_is_null(declared_return) && body_scope && + body_scope->self_name) { + const char *declared = python_annotation_chain(index, declared_return, 0); + const char *canonical = declared ? python_import_canonical(index, declared) : NULL; + TSNode parameters = field(function, "parameters"); + TSNode first = ts_node_named_child_count(parameters) ? ts_node_named_child(parameters, 0) + : (TSNode){0}; + const char *first_type = kind_is(first, "typed_parameter") + ? python_annotation_chain(index, field(first, "type"), 0) + : NULL; + if (declared && ((canonical && (!strcmp(canonical, "typing.Self") || + !strcmp(canonical, "typing_extensions.Self"))) || + (first_type && !strcmp(first_type, declared)))) { + const char **kept = cbm_arena_alloc(index->arena, 2 * sizeof(*kept)); + if (!kept) { + fail_index(index, CBM_RESOLVE_ANALYSIS_FAILED, "python_member_allocation"); + return; + } + kept[0] = body_scope->self_is_class + ? cbm_arena_sprintf(index->arena, "%s()", body_scope->self_name) + : body_scope->self_name; + kept[1] = NULL; + index->python_members[member].return_chain = NULL; + index->python_members[member].returned = kept; + index->python_members[member].returned_scope = scope; + return; + } + } + /* What __new__ and friends answer with decides whether they defer to the base, whatever + * they are annotated to return. */ + const char *name = node_text(index, field(function, "name")); + bool special = name && (!strcmp(name, "__new__") || !strcmp(name, "__getattribute__") || + !strcmp(name, "__class_getitem__")); + if (member >= index->python_member_count || + (!special && !ts_node_is_null(field(function, "return_type"))) || !python_gather_returns(index, field(function, "body"), chains, &count, RETURNS_MAX, 0) || !count) return; @@ -1665,6 +1674,185 @@ static const char *python_decorator_chain(cbm_resolve_binding_index_t *index, TS return python_chain(index, value, 0); } +static bool python_overload_stub(cbm_resolve_binding_index_t *index, TSNode decorated) { + uint32_t count = ts_node_named_child_count(decorated); + for (uint32_t i = 0; i < count; i++) { + TSNode child = ts_node_named_child(decorated, i); + if (!kind_is(child, "decorator") || !ts_node_named_child_count(child)) + continue; + const char *chain = python_chain(index, ts_node_named_child(child, 0), 0); + size_t length = chain ? strlen(chain) : 0; + if (length < 8 || strcmp(chain + length - 8, "overload")) + continue; + const char *canonical = python_import_canonical(index, chain); + if (canonical && (!strcmp(canonical, "typing.overload") || + !strcmp(canonical, "typing_extensions.overload"))) + return true; + } + return false; +} + +static void python_finish_narrows(cbm_resolve_binding_index_t *index); + +static void python_add_narrow(cbm_resolve_binding_index_t *index, TSNode test, uint32_t from, + uint32_t until) { + TSNode function = field(test, "function"), args = field(test, "arguments"); + const char *callee = kind_is(test, "call") && kind_is(function, "identifier") + ? node_text(index, function) + : NULL; + if (!callee || strcmp(callee, "isinstance") || ts_node_is_null(args) || + ts_node_named_child_count(args) != 2 || until <= from) + return; + TSNode kind = ts_node_named_child(args, 1); + const char *chain = python_chain(index, ts_node_named_child(args, 0), 0); + /* A tuple of classes says only that it is one of them. */ + const char *type = kind_is(kind, "identifier") || kind_is(kind, "attribute") + ? python_chain(index, kind, 0) + : NULL; + /* A call or subscript computes a new value each time. */ + if (!chain || !type || strpbrk(chain, "([@")) + return; + python_narrow_t *items = grow_array(index, index->python_narrows, index->python_narrow_count, + &index->python_narrow_capacity, sizeof(*items)); + if (!items) { + fail_index(index, CBM_RESOLVE_ANALYSIS_FAILED, "python_narrow_allocation"); + return; + } + index->python_narrows = items; + items[index->python_narrow_count++] = (python_narrow_t){ + .chain = chain, .type = type, .from = from, .until = until, .scope = index->current_scope}; +} + +/* Each conjunct of `a and b` holds in what follows it, up to the end of the block it guards. */ +static void python_note_condition(cbm_resolve_binding_index_t *index, TSNode condition, + uint32_t until, unsigned depth) { + if (kind_is(condition, "parenthesized_expression") && ts_node_named_child_count(condition) == 1) + condition = ts_node_named_child(condition, 0); + const char *operator= kind_is(condition, "boolean_operator") + ? node_text(index, field(condition, "operator")) + : NULL; + if (operator&& !strcmp(operator, "and") && depth < 8) { + python_note_condition(index, field(condition, "left"), until, depth + 1); + python_note_condition(index, field(condition, "right"), until, depth + 1); + return; + } + python_add_narrow(index, condition, ts_node_end_byte(condition), until); +} + +static uint32_t python_block_end(TSNode statement); + +/* isinstance as if and elif conditions, as `if not isinstance: leave` and as an assert. */ +static void python_note_narrowing(cbm_resolve_binding_index_t *index, TSNode node, + const char *kind) { + if (!strcmp(kind, "assert_statement")) { + TSNode test = ts_node_named_child_count(node) ? ts_node_named_child(node, 0) : (TSNode){0}; + python_add_narrow(index, test, ts_node_end_byte(node), python_block_end(node)); + return; + } + TSNode condition = field(node, "condition"), body = field(node, "consequence"); + if (ts_node_is_null(condition) || ts_node_is_null(body)) + return; + if (kind_is(condition, "not_operator")) { + /* The other way to reach what follows left the block. */ + if (!strcmp(kind, "if_statement") && python_block_always_exits(body, 0) && + ts_node_is_null(field(node, "alternative"))) + python_add_narrow(index, field(condition, "argument"), ts_node_end_byte(node), + python_block_end(node)); + return; + } + python_note_condition(index, condition, ts_node_end_byte(body), 0); +} + +/* Where the block holding this statement ends. Later statements of a block run only after + * this one. A def or lambda in between means the statement is not in such a block. */ +static uint32_t python_block_end(TSNode statement) { + TSNode node = statement; + for (unsigned depth = 0; depth < BINDING_MAX_DEPTH; depth++) { + node = ts_node_parent(node); + if (ts_node_is_null(node) || kind_is(node, "lambda")) + return 0; + if (kind_is(node, "block") || kind_is(node, "module")) + return ts_node_end_byte(node); + } + return 0; +} + +/* `with X() as y`: y holds what __enter__ returns, from there to the end of the with + * statement's block. `except E as e`: e holds an E, for that clause only. */ +static void python_note_as_target(cbm_resolve_binding_index_t *index, binding_name_t *name, + TSNode identifier) { + TSNode pattern = ts_node_parent(ts_node_parent(identifier)); + TSNode holder = ts_node_parent(pattern); + if (!kind_is(pattern, "as_pattern") || !ts_node_named_child_count(pattern)) + return; + const char *chain = python_chain(index, ts_node_named_child(pattern, 0), 0); + name->value_from = ts_node_end_byte(pattern); + if (kind_is(holder, "except_clause") || kind_is(holder, "except_group_clause")) { + name->value_chain = chain; + name->value_kind = CBM_PY_ROOT_TYPED; + name->value_until = ts_node_end_byte(holder); + } else if (kind_is(holder, "with_item")) { + TSNode statement = ts_node_parent(ts_node_parent(holder)); + bool awaited = kind_is(statement, "with_statement") && ts_node_child_count(statement) && + kind_is(ts_node_child(statement, 0), "async"); + name->value_chain = chain ? cbm_arena_sprintf(index->arena, "%s.%s()", chain, + awaited ? "__aenter__" : "__enter__") + : NULL; + name->value_kind = CBM_PY_ROOT_ASSIGNED; + name->value_until = python_block_end(statement); + } +} + +/* One name of a tuple a loop unpacks, when the iterable says what that position holds. */ +static void python_note_unpacked_loop_target(cbm_resolve_binding_index_t *index, + binding_name_t *name, TSNode identifier) { + TSNode pattern = ts_node_parent(identifier), loop = ts_node_parent(pattern); + if ((!kind_is(loop, "for_statement") && !kind_is(loop, "for_in_clause")) || + !ts_node_eq(pattern, field(loop, "left"))) + return; + uint32_t position = 0, count = ts_node_named_child_count(pattern); + bool located = false; + TSTreeCursor cursor = ts_tree_cursor_new(pattern); + for (bool more = ts_tree_cursor_goto_first_child(&cursor); more && !located; + more = ts_tree_cursor_goto_next_sibling(&cursor)) { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_eq(child, identifier)) + located = true; + else if (ts_node_is_named(child)) + position++; + } + ts_tree_cursor_delete(&cursor); + TSNode right = field(loop, "right"); + TSNode function = field(right, "function"), args = field(right, "arguments"); + if (!located || !kind_is(right, "call") || ts_node_is_null(args)) + return; + uint32_t supplied = ts_node_named_child_count(args); + const char *value = NULL; + if (kind_is(function, "attribute")) { + /* d.items() against an annotated map: the second is the value. */ + const char *member = node_text(index, field(function, "attribute")); + const char *object = python_chain(index, field(function, "object"), 0); + if (member && !strcmp(member, "items") && object && count == 2 && position == 1) + value = cbm_arena_sprintf(index->arena, "%s@values@iter", object); + } else if (kind_is(function, "identifier")) { + const char *callee = node_text(index, function); + const char *source = NULL; + if (callee && !strcmp(callee, "enumerate") && count == 2 && position == 1 && supplied >= 1) + source = python_chain(index, ts_node_named_child(args, 0), 0); + else if (callee && !strcmp(callee, "zip") && supplied == count) + source = python_chain(index, ts_node_named_child(args, position), 0); + /* The name is checked to be the builtin when this is read. */ + value = source ? cbm_arena_sprintf(index->arena, "%s(%s)@iter", callee, source) : NULL; + } + if (!value) + return; + name->value_chain = value; + name->value_kind = CBM_PY_ROOT_ASSIGNED; + name->value_from = ts_node_end_byte(right); + TSNode body = kind_is(loop, "for_statement") ? field(loop, "body") : ts_node_parent(loop); + name->value_until = ts_node_is_null(body) ? 0 : ts_node_end_byte(body); +} + /* What the name just recorded was bound to, when that is statically followable. * Lookup uses it only while this stays the scope's one binder of the name. */ static void python_note_binder_value(cbm_resolve_binding_index_t *index, TSNode identifier, @@ -1672,9 +1860,53 @@ static void python_note_binder_value(cbm_resolve_binding_index_t *index, TSNode binding_name_t *name = &index->names[index->name_count - 1]; uint32_t scope = index->current_scope; name->value_scope = scope; + if (parent && !strcmp(parent, "as_pattern_target")) { + python_note_as_target(index, name, identifier); + return; + } + + if (parent && + (!strcmp(parent, "parameters") || !strcmp(parent, "typed_parameter") || + !strcmp(parent, "typed_default_parameter") || !strcmp(parent, "default_parameter"))) + name->value_until = UINT32_MAX; + /* `def make(self, Request=Request)`: what the signature says it holds unless told + * otherwise, read where the def stands. */ + if (parent && !strcmp(parent, "default_parameter") && field_name && + !strcmp(field_name, "name") && scope) { + name->value_chain = python_chain(index, field(ts_node_parent(identifier), "value"), 0); + name->value_kind = name->value_chain ? CBM_PY_ROOT_ASSIGNED : name->value_kind; + name->value_scope = index->scopes[scope].method_of ? index->scopes[scope].method_of - 1 + : index->scopes[scope].parent; + return; + } + /* `for k, v in d.items()`, `for i, x in enumerate(E)`, `for a, b in zip(A, B)`. */ + if (parent && !strcmp(parent, "pattern_list")) { + python_note_unpacked_loop_target(index, name, identifier); + return; + } + /* `if (m := Foo()):` */ + if (parent && !strcmp(parent, "named_expression")) { + name->value_chain = python_chain(index, field(ts_node_parent(identifier), "value"), 0); + name->value_kind = name->value_chain ? CBM_PY_ROOT_ASSIGNED : name->value_kind; + return; + } + /* `for x in E`: x holds what E yields, for the loop's body. */ + if (parent && (!strcmp(parent, "for_statement") || !strcmp(parent, "for_in_clause")) && + field_name && !strcmp(field_name, "left")) { + TSNode loop = ts_node_parent(identifier), right = field(loop, "right"); + const char *chain = python_chain(index, right, 0); + TSNode body = !strcmp(parent, "for_statement") ? field(loop, "body") : ts_node_parent(loop); + name->value_chain = chain ? cbm_arena_sprintf(index->arena, "%s@iter", chain) : NULL; + name->value_kind = name->value_chain ? CBM_PY_ROOT_ASSIGNED : name->value_kind; + name->value_from = ts_node_end_byte(right); + name->value_until = ts_node_is_null(body) ? 0 : ts_node_end_byte(body); + return; + } if (parent && !strcmp(parent, "assignment") && field_name && !strcmp(field_name, "left")) { TSNode assignment = ts_node_parent(identifier); TSNode type = field(assignment, "type"), right = field(assignment, "right"); + name->value_from = ts_node_end_byte(assignment); + name->value_until = python_block_end(assignment); if (!ts_node_is_null(type)) { name->value_chain = python_annotation_chain(index, type, 0); name->value_kind = CBM_PY_ROOT_TYPED; @@ -1732,13 +1964,125 @@ static cbm_resolve_python_member_t *python_add_member(cbm_resolve_binding_index_ return entry->target ? entry : NULL; } +static bool python_constant_value(TSNode value) { + return kind_is(value, "integer") || kind_is(value, "float") || kind_is(value, "string") || + kind_is(value, "concatenated_string") || kind_is(value, "true") || + kind_is(value, "false") || kind_is(value, "none") || kind_is(value, "list") || + kind_is(value, "dictionary") || kind_is(value, "tuple") || kind_is(value, "set"); +} + +/* This body can only hand back a constant or nothing: a decorator like that discards the + * function. A stub says nothing, and neither does a body that only raises. The walk stays + * out of nested scopes, so across a file each node is visited once per enclosing def. */ +static bool python_body_discards(TSNode body) { + bool returned = false; + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool discards = true, more = ts_tree_cursor_goto_first_child(&cursor); + unsigned depth = 1; + while (more) { + TSNode node = ts_tree_cursor_current_node(&cursor); + bool nested = kind_is(node, "function_definition") || kind_is(node, "class_definition") || + kind_is(node, "decorated_definition") || kind_is(node, "lambda"); + if (kind_is(node, "yield")) { + /* A generator returns a generator, whatever its return says. */ + ts_tree_cursor_delete(&cursor); + return true; + } + if (kind_is(node, "return_statement")) { + returned = true; + if (ts_node_named_child_count(node) && + !python_constant_value(ts_node_named_child(node, 0))) { + discards = false; + break; + } + } + if (!nested && !kind_is(node, "return_statement") && + ts_tree_cursor_goto_first_child(&cursor)) { + depth++; + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (--depth == 0 || !ts_tree_cursor_goto_parent(&cursor)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + if (!discards || returned) + return discards; + if (python_block_always_exits(body, 0)) + return false; + uint32_t count = ts_node_named_child_count(body); + TSNode last = count ? ts_node_named_child(body, count - 1) : (TSNode){0}; + return !(kind_is(last, "expression_statement") && ts_node_named_child_count(last) && + kind_is(ts_node_named_child(last, 0), "ellipsis")); +} + +/* The name of a def's first parameter, when it has one a body could spell. */ +static const char *python_first_parameter(cbm_resolve_binding_index_t *index, TSNode function) { + TSNode parameters = field(function, "parameters"); + if (ts_node_is_null(parameters) || !ts_node_named_child_count(parameters)) + return NULL; + TSNode first = ts_node_named_child(parameters, 0); + if (kind_is(first, "typed_parameter") && ts_node_named_child_count(first)) + first = ts_node_named_child(first, 0); + else if (kind_is(first, "default_parameter") || kind_is(first, "typed_default_parameter")) + first = field(first, "name"); + return kind_is(first, "identifier") ? node_text(index, first) : NULL; +} + +static bool python_spells(const cbm_resolve_binding_index_t *index, TSNode node, const char *word) { + uint32_t start = ts_node_start_byte(node), end = ts_node_end_byte(node); + size_t length = strlen(word); + return end - start == length && end <= index->source_len && + !memcmp(index->source + start, word, length); +} + +/* `def replace(function): return other`. A body that returns a value and never spells its + * parameter cannot have built that value from the function. locals() and the like could reach + * it by another name. Stops at the first mention, and only top-level and nested defs get here, + * so across a file each node is visited once per enclosing def. */ +static bool python_body_replaces(cbm_resolve_binding_index_t *index, TSNode function) { + const char *parameter = python_first_parameter(index, function); + TSNode body = field(function, "body"); + if (!parameter || ts_node_is_null(body)) + return false; + bool returns = false, mentioned = false; + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool more = ts_tree_cursor_goto_first_child(&cursor); + unsigned depth = 1, nested = 0; + while (more && !mentioned) { + TSNode node = ts_tree_cursor_current_node(&cursor); + if (kind_is(node, "identifier")) + mentioned = python_spells(index, node, parameter) || + python_spells(index, node, "locals") || + python_spells(index, node, "vars") || python_spells(index, node, "eval") || + python_spells(index, node, "exec"); + returns = returns || + (!nested && kind_is(node, "return_statement") && ts_node_named_child_count(node)); + if (ts_tree_cursor_goto_first_child(&cursor)) { + depth++; + nested += kind_is(node, "function_definition") || kind_is(node, "lambda"); + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (--depth == 0 || !ts_tree_cursor_goto_parent(&cursor)) { + more = false; + break; + } + TSNode up = ts_tree_cursor_current_node(&cursor); + nested -= nested && (kind_is(up, "function_definition") || kind_is(up, "lambda")); + } + } + ts_tree_cursor_delete(&cursor); + return returns && !mentioned; +} + static void python_note_member_shape(cbm_resolve_binding_index_t *index, cbm_resolve_python_member_t *member, TSNode function) { - member->shape = python_decorator_shape(index, function, 0); - if (member->shape & CBM_PY_SHAPE_DELEGATES) { - member->delegate_chain = index->python_shape_delegate; - member->delegate_applied = index->python_shape_delegate_applied; - } + member->discards = python_body_discards(field(function, "body")); + member->replaces = !member->discards && python_body_replaces(index, function); } /* A def directly in a class body. Its decorators are classified with the file's @@ -1793,6 +2137,25 @@ static void python_note_function(cbm_resolve_binding_index_t *index, TSNode node python_note_member_shape(index, member, node); } +/* `go = classmethod(go)` is the decorator spelled out: the callee, when this is that shape. */ +static const char *python_spelled_decorator(cbm_resolve_binding_index_t *index, TSNode target, + const char *name) { + TSNode assignment = ts_node_parent(target); + if (!kind_is(assignment, "assignment")) + return NULL; + TSNode right = field(assignment, "right"); + TSNode function = field(right, "function"), args = field(right, "arguments"); + if (!kind_is(right, "call") || !kind_is(function, "identifier") || ts_node_is_null(args) || + ts_node_named_child_count(args) != 1) + return NULL; + TSNode only = ts_node_named_child(args, 0); + const char *callee = node_text(index, function); + return kind_is(only, "identifier") && python_spells(index, only, name) && callee && + (!strcmp(callee, "classmethod") || !strcmp(callee, "staticmethod")) + ? callee + : NULL; +} + /* Class-level assignment or declaration. No graph node carries it, but it stops * lookup: a base's method of the same name is not what this class exposes. */ static void python_note_class_value(cbm_resolve_binding_index_t *index, const char *name, TSNode at, @@ -1808,7 +2171,12 @@ static void python_note_class_value(cbm_resolve_binding_index_t *index, const ch else member->value_chain = chain; member->none_value = none; - member->chain_scope = owner; + member->spelled_decorator = python_spelled_decorator(index, at, name); + /* `ResultConsumer = ResultConsumer` in a class body reads the name from outside it: the + * binding does not exist until the statement has run. */ + size_t root = chain ? strcspn(chain, ".(") : 0; + bool self_named = root && strlen(name) == root && !strncmp(chain, name, root); + member->chain_scope = self_named ? index->scopes[owner].parent : owner; } /* The method whose first parameter a name in this scope chain denotes. */ @@ -1886,8 +2254,6 @@ static void python_note_class(cbm_resolve_binding_index_t *index, TSNode node, u for (uint32_t i = 0; i < count; i++) { TSNode base = ts_node_named_child(bases, i); if (kind_is(base, "keyword_argument")) { - const char *keyword = node_text(index, field(base, "name")); - entry->metaclass |= keyword && !strcmp(keyword, "metaclass"); continue; } if (kind_is(base, "comment")) @@ -3912,11 +4278,54 @@ static binding_walk_t note_node(cbm_resolve_binding_index_t *index, TSNode node, return state; } } + /* A star under TYPE_CHECKING states the module's names as its plain imports there do. */ if (index->language == CBM_LANG_PYTHON && !strcmp(kind, "wildcard_import") && - (inherited.scope || inherited.conditional)) { + (inherited.scope || (inherited.conditional && !python_type_checking_import(index, node)))) { fail_index(index, CBM_RESOLVE_ANALYSIS_UNSUPPORTED, "conditional_wildcard_import"); return state; } + if (index->language == CBM_LANG_PYTHON && + (!strcmp(kind, "if_statement") || !strcmp(kind, "elif_clause") || + !strcmp(kind, "assert_statement"))) + python_note_narrowing(index, node, kind); + /* A store through an attribute ends what a narrowing said about that attribute. */ + if (index->language == CBM_LANG_PYTHON && kind_is(node, "attribute") && field_name && + !strcmp(field_name, "left") && parent_kind && + (!strcmp(parent_kind, "assignment") || !strcmp(parent_kind, "augmented_assignment"))) { + const char *chain = python_chain(index, node, 0); + python_store_t *stores = + chain ? grow_array(index, index->python_stores, index->python_store_count, + &index->python_store_capacity, sizeof(*stores)) + : NULL; + if (stores) { + index->python_stores = stores; + stores[index->python_store_count++] = (python_store_t){ + .chain = chain, + .at = ts_node_end_byte(ts_node_parent(node)), + .scope = state.scope, + .value = strcmp(parent_kind, "assignment") + ? NULL + : python_chain(index, field(ts_node_parent(node), "right"), 0), + .value_at = ts_node_start_byte(node)}; + } + } + if (index->language == CBM_LANG_PYTHON && + (!strcmp(kind, "for_statement") || !strcmp(kind, "while_statement"))) { + python_loop_t *loops = grow_array(index, index->python_loops, index->python_loop_count, + &index->python_loop_capacity, sizeof(*loops)); + if (!loops) { + fail_index(index, CBM_RESOLVE_ANALYSIS_FAILED, "python_loop_allocation"); + return state; + } + index->python_loops = loops; + uint32_t start = ts_node_start_byte(node); + /* Loops nest or are disjoint, so the walk's order alone says which are still open. */ + while (index->python_open_loop && loops[index->python_open_loop - 1].end <= start) + index->python_open_loop = loops[index->python_open_loop - 1].parent; + loops[index->python_loop_count++] = (python_loop_t){ + .start = start, .end = ts_node_end_byte(node), .parent = index->python_open_loop}; + index->python_open_loop = (uint32_t)index->python_loop_count; + } bool import = kind_is(node, "import_statement") || kind_is(node, "import_from_statement"); if (import) { state.import = true; @@ -3934,6 +4343,12 @@ static binding_walk_t note_node(cbm_resolve_binding_index_t *index, TSNode node, note_import(index, node, true); } } + /* typing.overload stubs announce the def that follows: with it they are one declaration, + * and they bind nothing of their own. */ + bool overload_stub = index->language == CBM_LANG_PYTHON && + !strcmp(kind, "function_definition") && parent_kind && + !strcmp(parent_kind, "decorated_definition") && + python_overload_stub(index, ts_node_parent(node)); if (index->language == CBM_LANG_PYTHON && strcmp(kind, "class_definition") == 0) { /* A class creates a binding in its enclosing namespace. Its members * do not authorize callable targets, but must not erase independent @@ -3947,7 +4362,8 @@ static binding_walk_t note_node(cbm_resolve_binding_index_t *index, TSNode node, python_note_decorated(index, node, index->names[index->name_count - 1].target, class_name, true); } - if (strcmp(kind, "function_definition") == 0 || strcmp(kind, "function_declaration") == 0) { + if ((strcmp(kind, "function_definition") == 0 || strcmp(kind, "function_declaration") == 0) && + !overload_stub) { TSNode name = field(node, "name"); /* One binder proves identity wherever it is bound at all, so a def under * a condition still counts; an alternative def makes two binders. */ @@ -3973,9 +4389,9 @@ static binding_walk_t note_node(cbm_resolve_binding_index_t *index, TSNode node, uint32_t enclosing_scope = state.scope; bool in_class = enclosing_scope && index->scopes[enclosing_scope].class_scope; size_t function_member = index->python_member_count; - if (in_class && !strcmp(kind, "function_definition")) + if (in_class && !strcmp(kind, "function_definition") && !overload_stub) python_note_method(index, node, parent_kind, enclosing_scope, state.conditional); - else if (!strcmp(kind, "function_definition")) + else if (!strcmp(kind, "function_definition") && !overload_stub) python_note_function(index, node, enclosing_scope); if (in_class && !strcmp(kind, "class_definition")) python_note_class_value(index, node_text(index, field(node, "name")), node, NULL, false, @@ -3985,7 +4401,7 @@ static binding_walk_t note_node(cbm_resolve_binding_index_t *index, TSNode node, python_child_scope(index, node, enclosing_scope, !strcmp(kind, "class_definition")); if (!strcmp(kind, "class_definition")) python_note_class(index, node, state.scope, enclosing_scope); - if (!strcmp(kind, "function_definition")) + if (!strcmp(kind, "function_definition") && !overload_stub) python_note_returns(index, node, function_member, state.scope); state.header_scope = enclosing_scope; state.conditional = false; @@ -4688,47 +5104,102 @@ static cbm_resolve_result_t binding_query_finish(const cbm_resolve_binding_index return result; } -/* A module value escaping to an unknown alias/callee can mutate its namespace. - * This independent pass still visits syntax outside the positive-binding - * subset. Imported-name queries are binary searches, never source rescans. */ -static bool python_namespace_read(cbm_resolve_binding_index_t *index, TSNode node) { +typedef enum { PY_NS_READ, PY_NS_WRITE, PY_NS_ANY } python_namespace_use_t; + +/* A plain string's text, NULL for anything computed. */ +static const char *python_plain_string(cbm_resolve_binding_index_t *index, TSNode node) { + if (!kind_is(node, "string") || ts_node_named_child_count(node) != 3) + return NULL; + TSNode content = ts_node_named_child(node, 1); + return kind_is(content, "string_content") ? node_text(index, content) : NULL; +} + +/* What one use of a name does to the module it may hold. A namespace changes by a + * statement that stores or deletes through it, and that statement names what it changes. + * Reading a member, calling, or handing the module to a call selects no other + * declaration: what a callee does with it is not a static fact. __dict__, vars() and a + * computed setattr can reach any name. Queries are binary searches, never rescans. */ +static python_namespace_use_t python_namespace_use(cbm_resolve_binding_index_t *index, TSNode node, + const char **written) { TSNode parent = ts_node_parent(node); - if (kind_is(parent, "attribute") && ts_node_eq(node, field(parent, "attribute"))) - return true; /* Attribute spelling is not a lexical module reference. */ - if (kind_is(parent, "call") && ts_node_eq(node, field(parent, "function"))) - return true; /* Calling a module is an error, not an alias escape. */ if (kind_is(parent, "argument_list")) { - /* These builtins look at their argument and keep nothing of it. */ - static const char *const readers[] = {"getattr", "hasattr", "isinstance", "issubclass", - "dir", "repr", "str", "id", - "type", "callable", "print", NULL}; TSNode call = ts_node_parent(parent); const char *callee = node_text(index, field(call, "function")); - return callee && listed(callee, readers); + if (callee && !strcmp(callee, "vars")) + return PY_NS_ANY; + if (!callee || (strcmp(callee, "setattr") && strcmp(callee, "delattr")) || + !ts_node_eq(node, ts_node_named_child(parent, 0))) + return PY_NS_READ; + + *written = python_plain_string(index, ts_node_named_child(parent, 1)); + return *written ? PY_NS_WRITE : PY_NS_ANY; } if (!kind_is(parent, "attribute") || !ts_node_eq(node, field(parent, "object"))) - return kind_is(parent, "expression_statement"); - /* Reading a member hands out that member, not the module it came from. Only - * storing through the path, or asking for __dict__, can reach the namespace. */ + return PY_NS_READ; + const char *path = NULL; for (unsigned depth = 0; depth < BINDING_MAX_DEPTH; depth++) { const char *member = node_text(index, field(parent, "attribute")); if (!member || !strcmp(member, "__dict__")) - return false; + return PY_NS_ANY; + path = path ? cbm_arena_sprintf(index->arena, "%s.%s", path, member) : member; node = parent; parent = ts_node_parent(node); if (kind_is(parent, "attribute") && ts_node_eq(node, field(parent, "object"))) continue; - if ((kind_is(parent, "assignment") || kind_is(parent, "augmented_assignment")) && - ts_node_eq(node, field(parent, "left"))) - return false; - return !kind_is(parent, "delete_statement") && !kind_is(parent, "pattern_list") && - !kind_is(parent, "as_pattern_target"); + bool stored = ((kind_is(parent, "assignment") || kind_is(parent, "augmented_assignment")) && + ts_node_eq(node, field(parent, "left"))) || + kind_is(parent, "delete_statement") || + (kind_is(parent, "expression_list") && + kind_is(ts_node_parent(parent), "delete_statement")) || + kind_is(parent, "pattern_list") || kind_is(parent, "as_pattern_target"); + if (!stored) + return PY_NS_READ; + *written = path; + return path ? PY_NS_WRITE : PY_NS_ANY; } - return false; + return PY_NS_ANY; } -static int python_escape_compare(const void *left, const void *right) { - return strcmp(*(const char *const *)left, *(const char *const *)right); + +static int python_write_compare(const void *left, const void *right) { + return strcmp(((const python_write_t *)left)->name, ((const python_write_t *)right)->name); } + +/* `alias = pkg; alias.child = None` writes through pkg. Two rounds follow an alias of an alias; + * each is one search per alias statement. */ +static void python_follow_write_aliases(cbm_resolve_binding_index_t *index) { + for (unsigned round = 0; round < 2; round++) { + size_t sorted = index->python_write_count; + for (size_t a = 0; a < index->python_alias_count; a++) { + const python_alias_t alias = index->python_aliases[a]; + size_t low = 0, high = sorted; + while (low < high) { + size_t middle = low + (high - low) / 2; + if (strcmp(index->python_writes[middle].name, alias.alias) < 0) + low = middle + 1; + else + high = middle; + } + for (; low < sorted && !strcmp(index->python_writes[low].name, alias.alias); low++) { + const char *member = index->python_writes[low].member; + python_write_t *writes = + grow_array(index, index->python_writes, index->python_write_count, + &index->python_write_capacity, sizeof(*writes)); + if (!writes) { + index->python_writes_unseen = true; + return; + } + index->python_writes = writes; + writes[index->python_write_count++] = + (python_write_t){.name = alias.of, .member = member}; + } + } + if (index->python_write_count == sorted) + return; + qsort(index->python_writes, index->python_write_count, sizeof(*index->python_writes), + python_write_compare); + } +} + static void python_namespace_effects(cbm_resolve_binding_index_t *index, TSNode root) { TSTreeCursor cursor = ts_tree_cursor_new(root); unsigned depth = 0; @@ -4738,6 +5209,7 @@ static void python_namespace_effects(cbm_resolve_binding_index_t *index, TSNode TSNode node = ts_tree_cursor_current_node(&cursor); if (++visited > BINDING_MAX_NODES || depth > BINDING_MAX_DEPTH) { index->python_effects_complete = false; + index->python_writes_unseen = true; break; } bool skip = kind_is(node, "import_statement") || kind_is(node, "import_from_statement"); @@ -4752,17 +5224,42 @@ static void python_namespace_effects(cbm_resolve_binding_index_t *index, TSNode python_namespace_call_writes(index, node, name)) index->python_effects_complete = false; } - if (kind_is(node, "identifier") && !python_namespace_read(index, node)) { - const char *name = node_text(index, node); - const char **names = - grow_array(index, index->python_escapes, index->python_escape_count, - &index->python_escape_capacity, sizeof(*names)); - if (!name || !names) { + if (kind_is(node, "identifier") && !member && kind_is(parent, "argument_list") && + python_spells(index, node, "__name__")) + index->python_name_handed = true; + if (kind_is(node, "identifier") && !member) { + const char *written = NULL; + python_namespace_use_t use = python_namespace_use(index, node, &written); + const char *name = use != PY_NS_READ ? node_text(index, node) : NULL; + python_write_t *writes = + use != PY_NS_READ + ? grow_array(index, index->python_writes, index->python_write_count, + &index->python_write_capacity, sizeof(*writes)) + : NULL; + if (use != PY_NS_READ && (!name || !writes)) { index->python_effects_complete = false; + index->python_writes_unseen = true; break; } - index->python_escapes = names; - names[index->python_escape_count++] = name; + if (use != PY_NS_READ) { + index->python_writes = writes; + writes[index->python_write_count++] = + (python_write_t){.name = name, .member = use == PY_NS_WRITE ? written : NULL}; + } else if (kind_is(parent, "assignment") && ts_node_eq(node, field(parent, "right")) && + kind_is(field(parent, "left"), "identifier")) { + python_alias_t *aliases = + grow_array(index, index->python_aliases, index->python_alias_count, + &index->python_alias_capacity, sizeof(*aliases)); + const char *alias = node_text(index, field(parent, "left")); + const char *of = node_text(index, node); + if (!aliases || !alias || !of) { + index->python_effects_complete = false; + index->python_writes_unseen = true; + break; + } + index->python_aliases = aliases; + aliases[index->python_alias_count++] = (python_alias_t){alias, of}; + } } if (!skip && ts_tree_cursor_goto_first_child(&cursor)) { depth++; @@ -4776,9 +5273,10 @@ static void python_namespace_effects(cbm_resolve_binding_index_t *index, TSNode } finished: ts_tree_cursor_delete(&cursor); - if (index->python_escape_count > 1) - qsort(index->python_escapes, index->python_escape_count, sizeof(*index->python_escapes), - python_escape_compare); + if (index->python_write_count > 1) + qsort(index->python_writes, index->python_write_count, sizeof(*index->python_writes), + python_write_compare); + python_follow_write_aliases(index); } /* Symbol QNs omit a terminal index filename, while module QNs retain it. @@ -4962,6 +5460,7 @@ static cbm_resolve_binding_index_t *binding_build(CBMArena *arena, const CBMFile walk_tree(index, ts_tree_root_node(tree)); if (language == CBM_LANG_PYTHON) { python_namespace_effects(index, ts_tree_root_node(tree)); + python_finish_narrows(index); } } if (index->analysis == CBM_RESOLVE_ANALYSIS_COMPLETE && javascript_language(language) && @@ -5781,13 +6280,24 @@ bool cbm_resolve_binding_python_effects_complete(const cbm_resolve_binding_index return index && index->language == CBM_LANG_PYTHON && index->python_effects_complete; } -bool cbm_resolve_binding_python_import_escaped(const cbm_resolve_binding_index_t *index, - const char *name) { - if (!index || !name || !index->python_effects_complete) - return true; - return index->python_escape_count && - bsearch(&name, index->python_escapes, index->python_escape_count, - sizeof(*index->python_escapes), python_escape_compare) != NULL; +bool cbm_resolve_binding_python_import_write(const cbm_resolve_binding_index_t *index, + const char *name, size_t offset, const char **member) { + *member = NULL; + if (!index || !name || index->python_writes_unseen) + return offset == 0; + size_t low = 0, high = index->python_write_count; + while (low < high) { + size_t middle = low + (high - low) / 2; + if (strcmp(index->python_writes[middle].name, name) < 0) + low = middle + 1; + else + high = middle; + } + if (offset >= index->python_write_count - low || + strcmp(index->python_writes[low + offset].name, name)) + return false; + *member = index->python_writes[low + offset].member; + return true; } /* A completed audit saw every statement that binds at module level, including @@ -5800,7 +6310,14 @@ bool cbm_resolve_binding_python_namespace_closed(const cbm_resolve_binding_index index->analysis != CBM_RESOLVE_ANALYSIS_COMPLETE) return false; (void)find_scoped_name(index, "__getattr__", strlen("__getattr__"), 0, &present, JS_SPACE_ANY); - return !present; + return !present && !index->python_namespace_open; +} +/* Names this module binds nothing for can still be there at run time: __getattr__, a computed + * store into globals(), or handing its own __name__ on (celery's recreate_module, + * lazy_loader.attach). */ +bool cbm_resolve_binding_python_names_come_at_run_time(const cbm_resolve_binding_index_t *index) { + return index && + (!cbm_resolve_binding_python_namespace_closed(index) || index->python_name_handed); } size_t cbm_resolve_binding_python_namespace_size(const cbm_resolve_binding_index_t *index) { return index && index->language == CBM_LANG_PYTHON ? index->name_count : 0; @@ -5852,6 +6369,251 @@ const char *cbm_resolve_binding_python_method_owner(const cbm_resolve_binding_in return NULL; return index->scopes[index->scopes[scope].method_of - 1].qualified_name; } +static int python_narrow_compare(const void *left, const void *right) { + const python_narrow_t *a = left, *b = right; + if (a->from != b->from) + return a->from < b->from ? -1 : 1; + return a->until == b->until ? 0 : a->until > b->until ? -1 : 1; +} + +static int python_store_compare(const void *left, const void *right) { + const python_store_t *a = left, *b = right; + int order = strcmp(a->chain, b->chain); + if (order) + return order; + return a->at == b->at ? 0 : a->at < b->at ? -1 : 1; +} + +/* Sorted once after the walk. Regions nest or are disjoint, so one pass with the chain of open + * regions finds each one's parent, and a site reaches its enclosing regions by those links. */ +static void python_finish_narrows(cbm_resolve_binding_index_t *index) { + if (index->python_narrow_count > 1) + qsort(index->python_narrows, index->python_narrow_count, sizeof(*index->python_narrows), + python_narrow_compare); + if (index->python_store_count > 1) + qsort(index->python_stores, index->python_store_count, sizeof(*index->python_stores), + python_store_compare); + uint32_t open = 0; + for (size_t i = 0; i < index->python_narrow_count; i++) { + python_narrow_t *narrow = &index->python_narrows[i]; + while (open && index->python_narrows[open - 1].until < narrow->from) + open = index->python_narrows[open - 1].parent; + narrow->parent = open; + open = (uint32_t)i + 1; + } +} + +/* Any store of exactly this attribute chain between two positions. */ +static bool python_attribute_stored(const cbm_resolve_binding_index_t *index, const char *chain, + uint32_t from, uint32_t to) { + size_t low = 0, high = index->python_store_count; + while (low < high) { + size_t mid = low + (high - low) / 2; + int order = strcmp(index->python_stores[mid].chain, chain); + if (order < 0 || (!order && index->python_stores[mid].at < from)) + low = mid + 1; + else + high = mid; + } + return low < index->python_store_count && !strcmp(index->python_stores[low].chain, chain) && + index->python_stores[low].at <= to; +} + +/* Any binder of this local between two positions. */ +static bool python_local_rebound(const cbm_resolve_binding_index_t *index, const char *name, + size_t length, uint32_t scope, uint32_t from, uint32_t to) { + size_t low = 0, high = index->name_count; + while (low < high) { + size_t mid = low + (high - low) / 2; + int compare = strncmp(index->names[mid].name, name, length); + if (!compare && index->names[mid].name[length]) + compare = 1; + if (!compare) + compare = (index->names[mid].scope > scope) - (index->names[mid].scope < scope); + if (compare < 0) + low = mid + 1; + else + high = mid; + } + for (size_t i = low; + i < index->name_count && index->names[i].scope == scope && + strlen(index->names[i].name) == length && !strncmp(index->names[i].name, name, length); + i++) + if (index->names[i].at >= from && index->names[i].at <= to) + return true; + return false; +} + +/* A store of this exact chain finished before the site, in the scope holding the site or one + * around it. Scopes nest as text does, so when the site's own scope has such a store the nearest + * store before the site lies in it or inside it: one search, two walks up the nesting. */ +static const python_store_t *python_stored_earlier(const cbm_resolve_binding_index_t *index, + const char *chain, size_t length, uint32_t scope, + uint32_t site) { + size_t low = 0, high = index->python_store_count; + while (low < high) { + size_t mid = low + (high - low) / 2; + const python_store_t *store = &index->python_stores[mid]; + int order = strncmp(store->chain, chain, length); + if (!order && store->chain[length]) + order = 1; + if (order < 0 || (!order && store->at <= site)) + low = mid + 1; + else + high = mid; + } + if (!low) + return NULL; + const python_store_t *store = &index->python_stores[low - 1]; + if (strncmp(store->chain, chain, length) || store->chain[length]) + return NULL; + uint32_t inside = store->scope, around = scope; + for (unsigned depth = 0; depth < BINDING_MAX_DEPTH; depth++) { + if (inside == scope || around == store->scope) + return store; + if (!inside && !around) + return NULL; + inside = inside ? index->scopes[inside].parent : 0; + around = around ? index->scopes[around].parent : 0; + } + return NULL; +} + +/* `t.x = make()` says what t.x holds from there on, whatever class t.x is declared in, for what + * is read through it: t.x itself stays the member it names. Stores through a method's own first + * parameter are read as the class's instance attributes, so only what lies past its first + * attribute counts for that root. The longest such prefix of the chain answers. */ +bool cbm_resolve_binding_python_store_precedes(const cbm_resolve_binding_index_t *index, + const char *chain, uint32_t scope, uint32_t site, + size_t *prefix, const char **value, + uint32_t *value_at, uint32_t *value_scope) { + if (!index || !index->python_store_count) + return false; + size_t at = strcspn(chain, ".([@"); + cbm_resolve_python_root_kind_t root = + cbm_resolve_binding_python_root(index, chain, at, scope, site).kind; + bool own = root == CBM_PY_ROOT_SELF_PARAMETER || root == CBM_PY_ROOT_CLASS_PARAMETER; + bool found = false; + for (bool first = true; chain[at] == '.'; first = false) { + at += 1 + strcspn(chain + at + 1, ".([@"); + const python_store_t *store = chain[at] && !(first && own) + ? python_stored_earlier(index, chain, at, scope, site) + : NULL; + if (store) { + found = true; + *prefix = at; + *value = store->value; + *value_at = store->value_at; + *value_scope = store->scope; + } + } + return found; +} + +const char *cbm_resolve_binding_python_narrowed(const cbm_resolve_binding_index_t *index, + const char *chain, uint32_t scope, uint32_t site, + size_t *consumed) { + if (!index || !index->python_narrow_count) + return NULL; + size_t low = 0, high = index->python_narrow_count; + while (low < high) { + size_t mid = low + (high - low) / 2; + if (index->python_narrows[mid].from <= site) + low = mid + 1; + else + high = mid; + } + for (uint32_t at = (uint32_t)low; at; at = index->python_narrows[at - 1].parent) { + const python_narrow_t *narrow = &index->python_narrows[at - 1]; + size_t length = strlen(narrow->chain); + if (narrow->until < site || narrow->scope != scope || + strncmp(chain, narrow->chain, length) || + (chain[length] && !strchr(".([@", chain[length]))) + continue; + size_t root = strcspn(narrow->chain, ".([@"); + if (python_local_rebound(index, narrow->chain, root, scope, narrow->from, site) || + (narrow->chain[root] && + python_attribute_stored(index, narrow->chain, narrow->from, site))) + return NULL; + *consumed = length; + return narrow->type; + } + return NULL; +} + +/* A later binder inside a loop that holds the site but began after this binder comes back + * round to the site without passing it. Nesting is what bounds the hops. */ +static bool python_loop_comes_back(const cbm_resolve_binding_index_t *index, uint32_t binder, + uint32_t site, uint32_t later) { + size_t low = 0, high = index->python_loop_count; + while (low < high) { + size_t mid = low + (high - low) / 2; + if (index->python_loops[mid].start <= site) + low = mid + 1; + else + high = mid; + } + for (uint32_t at = (uint32_t)low; at;) { + const python_loop_t *loop = &index->python_loops[at - 1]; + if (loop->start <= binder) + return false; + if (loop->end >= site && loop->end >= later) + return true; + at = loop->parent; + } + return false; +} + +cbm_resolve_python_flow_t cbm_resolve_binding_python_reaching( + const cbm_resolve_binding_index_t *index, const char *name, size_t length, uint32_t scope, + uint32_t site, const char **chain, uint32_t *at, uint32_t *chain_scope, bool *typed) { + if (!index || index->language != CBM_LANG_PYTHON || + index->analysis != CBM_RESOLVE_ANALYSIS_COMPLETE || !scope || scope >= index->scope_count || + index->scopes[scope].class_scope) + return CBM_PY_FLOW_NONE; + size_t low = 0, high = index->name_count; + while (low < high) { + size_t mid = low + (high - low) / 2; + int compare = strncmp(index->names[mid].name, name, length); + if (!compare && index->names[mid].name[length]) + compare = 1; + if (!compare) + compare = (index->names[mid].scope > scope) - (index->names[mid].scope < scope); + if (compare < 0) + low = mid + 1; + else + high = mid; + } + const binding_name_t *best = NULL, *later = NULL; + size_t count = 0; + for (; low + count < index->name_count && index->names[low + count].scope == scope && + strlen(index->names[low + count].name) == length && + !strncmp(index->names[low + count].name, name, length); + count++) { + const binding_name_t *binder = &index->names[low + count]; + uint32_t from = binder->value_from ? binder->value_from : binder->at; + if (from <= site) { + if (!best || binder->at > best->at) + best = binder; + } else if (!later || binder->at < later->at) { + later = binder; + } + if (count > 64) + return CBM_PY_FLOW_NONE; + } + if (count < 2 || !best || best->value_until < site || + (later && python_loop_comes_back(index, best->at, site, later->at))) + return CBM_PY_FLOW_NONE; + if (best->flags != BINDING_BLOCKED || best->value_none || !best->value_chain || + (best->value_kind != CBM_PY_ROOT_ASSIGNED && best->value_kind != CBM_PY_ROOT_TYPED)) + return CBM_PY_FLOW_UNKNOWN; + *chain = best->value_chain; + *at = best->at; + *chain_scope = best->value_scope; + *typed = best->value_kind == CBM_PY_ROOT_TYPED; + return CBM_PY_FLOW_VALUE; +} + /* A local rebound only by plain assignments: x = None, then x = Store(). A None * says nothing about the class a member read finds. Anything else that binds * the name leaves it unknown. */ diff --git a/internal/cbm/resolve/binding.h b/internal/cbm/resolve/binding.h index 68f572615..5b5c650ab 100644 --- a/internal/cbm/resolve/binding.h +++ b/internal/cbm/resolve/binding.h @@ -80,14 +80,6 @@ typedef enum { CBM_PY_MEMBER_OPAQUE /* conditionally defined: the name stops lookup */ } cbm_resolve_python_member_kind_t; -/* What a function does when used as a decorator, proven from its source. */ -enum { - CBM_PY_SHAPE_DIRECT = 1u << 0, - CBM_PY_SHAPE_FACTORY = 1u << 1, - /* It hands the function to another decorator, named by delegate_chain. */ - CBM_PY_SHAPE_DELEGATES = 1u << 2 -}; - typedef struct { uint32_t owner_scope, chain_scope, at; const char *owner, *name, *target; @@ -95,9 +87,9 @@ typedef struct { const char *const *decorators; const char *return_chain, *value_chain, *type_chain; bool none_value; - unsigned shape; - const char *delegate_chain; - bool delegate_applied; + bool discards; /* as a decorator: returns a constant or nothing on every path */ + bool replaces; /* as a decorator: returns a value it never built from its parameter */ + const char *spelled_decorator; /* `go = classmethod(go)`: the callee */ /* What the function returns, as access paths in its own scope. NULL when a * return is not followable or the function is a generator. */ const char *const *returned; @@ -129,6 +121,7 @@ typedef struct { typedef struct { const char *module_path; uint32_t at; + bool annotation_only; /* under `if TYPE_CHECKING:` */ } cbm_resolve_python_star_t; /* Build once per source file. Borrows result/source until the arena is freed. @@ -286,8 +279,38 @@ typedef struct { * non-module scopes; copy while the index is alive. */ /* Dynamic execution/module registry access remains an unknown project effect. */ bool cbm_resolve_binding_python_effects_complete(const cbm_resolve_binding_index_t *); -/* Conservatively includes writes and unknown alias/callee escapes. */ -bool cbm_resolve_binding_python_import_escaped(const cbm_resolve_binding_index_t *, const char *); +/* A type annotation's chain as a value chain: `X` is X(), `X@seq` is X()@seq, `X@type` is X. */ +const char *cbm_resolve_python_typed_text(CBMArena *arena, const char *type, const char *rest); + +/* The class an isinstance test makes the longest narrowed prefix of this chain at this site. + * *consumed is how much of the chain it covers. NULL when no test applies. */ +const char *cbm_resolve_binding_python_narrowed(const cbm_resolve_binding_index_t *index, + const char *chain, uint32_t scope, uint32_t site, + size_t *consumed); +/* A store through a proper prefix of this chain earlier in the scope holding the site: that + * prefix's length, and the chain it stored (NULL when not followable) with where to read it. */ +bool cbm_resolve_binding_python_store_precedes(const cbm_resolve_binding_index_t *index, + const char *chain, uint32_t scope, uint32_t site, + size_t *prefix, const char **value, + uint32_t *value_at, uint32_t *value_scope); + +/* Which binder of a local every path to this site passes last. VALUE names its value, NONE + * means the flow does not decide it, UNKNOWN that it does and its value cannot be followed. */ +typedef enum { + CBM_PY_FLOW_NONE = 0, + CBM_PY_FLOW_VALUE, + CBM_PY_FLOW_UNKNOWN +} cbm_resolve_python_flow_t; +cbm_resolve_python_flow_t cbm_resolve_binding_python_reaching( + const cbm_resolve_binding_index_t *index, const char *name, size_t length, uint32_t scope, + uint32_t site, const char **chain, uint32_t *at, uint32_t *chain_scope, bool *typed); + +/* The offset-th write this file makes through a module it bound to name. *member is the + * dotted path written below that module, NULL when any name can change. False past the + * last one. A read, a call and a hand-off to a callee are not writes. */ +bool cbm_resolve_binding_python_import_write(const cbm_resolve_binding_index_t *, const char *name, + size_t offset, const char **member); +bool cbm_resolve_binding_python_names_come_at_run_time(const cbm_resolve_binding_index_t *index); bool cbm_resolve_binding_python_namespace_closed(const cbm_resolve_binding_index_t *index); size_t cbm_resolve_binding_python_namespace_size(const cbm_resolve_binding_index_t *index); const char *cbm_resolve_binding_python_namespace_name(const cbm_resolve_binding_index_t *index, diff --git a/internal/cbm/resolve/capabilities.c b/internal/cbm/resolve/capabilities.c index 8d2a20b65..09cc193fb 100644 --- a/internal/cbm/resolve/capabilities.c +++ b/internal/cbm/resolve/capabilities.c @@ -302,10 +302,11 @@ bool cbm_resolve_capabilities(CBMLanguage language, cbm_resolve_capabilities_t * CBM_RESOLVE_CAP_DECLARED_TYPE | CBM_RESOLVE_CAP_MEMBER | CBM_RESOLVE_CAP_TYPE_FLOW | CBM_RESOLVE_CAP_CALLABLE; out->limitation = - "Imports, reexports and closed star imports; self, cls and super through C3 order; " - "receivers typed by construction or annotation; decorators proven to keep identity. " - "Untyped receivers, computed bases, class-body values a metaclass can replace and " - "unproven decorators abstain."; + "Imports, reexports and closed star imports, through source roots a manifest states; " + "self, cls and super through C3 order; receivers typed by construction, annotation, " + "isinstance, with or except; annotated containers; a decorator keeps the declaration " + "unless its source discards the function. Untyped receivers, computed bases, Enum " + "and NamedTuple members and classes reached by computed writes abstain."; break; case CBM_LANG_PHP: out->adapter = "php_source"; diff --git a/internal/cbm/resolve/py_members.c b/internal/cbm/resolve/py_members.c index 6c8cb9e57..a0db10cfc 100644 --- a/internal/cbm/resolve/py_members.c +++ b/internal/cbm/resolve/py_members.c @@ -2,6 +2,7 @@ #include "foundation/hash_table.h" #include "foundation/mem_core.h" #include + #include /* Finite analysis per occurrence. Past either limit the receiver is unknown, the @@ -9,6 +10,9 @@ * other would otherwise branch without end. */ enum { PY_MAX_DEPTH = 24, PY_MAX_MRO = 512, PY_MAX_REF_STEPS = 64, PY_MAX_EVAL_STEPS = 2048 }; +/* Where a chain's root name ends, and each step after it. */ +#define PY_CHAIN_STOPS ".([@" + typedef enum { PY_REF_NONE = 0, PY_REF_KEY, /* a shared solver fact */ @@ -65,28 +69,31 @@ typedef struct py_class py_class_t; typedef struct { py_ref_t result; - unsigned shape; - py_ref_t delegate; - bool delegate_applied; + bool discards; /* as a decorator: returns a constant or nothing */ + bool replaces; /* as a bare decorator: returns a value not built from its parameter */ /* Unannotated: the class every return statement agrees on, settled at seal. */ py_ref_t *returned; uint32_t returned_count; const py_class_t *inferred; + bool returns_class; /* inferred is the class object itself, not an instance */ + bool returns_receiver; /* every return is self, or cls(...): the receiver's class */ unsigned char inference; /* 0 pending, 1 in progress, 2 settled */ } py_function_t; typedef struct { const char *name, *target; py_member_kind_t kind; - py_ref_t value; /* return type, property type, or class attribute value */ - bool none_default; /* `x = None` in the class body: instances store the real value */ - bool annotated; /* its value comes from an annotation */ + py_ref_t value; /* return type, property type, or class attribute value */ + bool none_default; /* `x = None` in the class body: instances store the real value */ + bool spelled_decorator; /* `go = classmethod(go)`, with the builtin */ + bool annotated; /* its value comes from an annotation */ } py_member_t; typedef struct { const char *name; py_ref_t value; bool none; + bool annotated; /* `self.x: T = ...` or `x: T` in the body */ } py_attribute_t; struct py_class { @@ -174,7 +181,7 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL}; if (!chain || depth > PY_MAX_DEPTH || ++*steps > PY_MAX_REF_STEPS) return none; - size_t length = strcspn(chain, ".("); + size_t length = strcspn(chain, PY_CHAIN_STOPS); const char *rest = chain + length; /* super and type are the builtins only while nothing in scope binds the name. */ bool builtin = rest[0] == '(' && @@ -209,8 +216,19 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre const char *open = strchr(chain, '('); if (open && open[1] != ')' && open > chain) { const char *spelled = cbm_arena_strndup(arena, chain, (size_t)(open - chain)); - if (spelled && (!strcmp(spelled, "partial") || !strcmp(spelled, "functools.partial")) && - cbm_resolve_binding_python_import_is(index, spelled, "functools.partial")) { + bool partial = spelled && + (!strcmp(spelled, "partial") || !strcmp(spelled, "functools.partial")) && + cbm_resolve_binding_python_import_is(index, spelled, "functools.partial"); + /* list(x), sorted(x), reversed(x) hold what x holds, while they are the builtins. */ + bool passing = + spelled && + (!strcmp(spelled, "list") || !strcmp(spelled, "tuple") || !strcmp(spelled, "set") || + !strcmp(spelled, "frozenset") || !strcmp(spelled, "sorted") || + !strcmp(spelled, "reversed") || !strcmp(spelled, "iter") || + !strcmp(spelled, "enumerate") || !strcmp(spelled, "zip")) && + cbm_resolve_binding_python_root(index, spelled, strlen(spelled), scope, at).kind == + CBM_PY_ROOT_UNBOUND; + if (partial || passing) { int nesting = 0; const char *close = open; for (; *close; close++) { @@ -218,10 +236,13 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre if (!nesting) break; } - if (!*close || close[1]) + if (!*close || (partial && close[1])) return none; - const char *wrapped = cbm_arena_strndup(arena, open + 1, (size_t)(close - open - 1)); - return py_ref_of_chain(index, arena, imports, wrapped, scope, at, depth + 1, steps); + const char *wrapped = + cbm_arena_sprintf(arena, "%.*s%s", (int)(close - open - 1), open + 1, close + 1); + return wrapped ? py_ref_of_chain(index, arena, imports, wrapped, scope, at, depth + 1, + steps) + : none; } } cbm_resolve_python_root_t root = @@ -244,7 +265,8 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre return none; for (uint32_t i = 0; i < root.chain_count; i++) { const char *arm = root.chains[i]; - if (!strncmp(arm, chain, length) && (!arm[length] || strchr(".(", arm[length]))) { + if (!strncmp(arm, chain, length) && + (!arm[length] || strchr(PY_CHAIN_STOPS, arm[length]))) { tails[join->self_count++] = arm + length; continue; } @@ -296,10 +318,10 @@ static py_ref_t py_ref_of_type(const cbm_resolve_binding_index_t *index, CBMAren py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL}; if (!chain) return none; - size_t length = strcspn(chain, ".("); + size_t length = strcspn(chain, PY_CHAIN_STOPS); cbm_resolve_python_root_t root = cbm_resolve_binding_python_root(index, chain, length, scope, 0); - const char *typed = cbm_arena_sprintf(arena, "%s()%s", chain, rest ? rest : ""); + const char *typed = cbm_resolve_python_typed_text(arena, chain, rest ? rest : ""); if (!typed) return none; if (root.kind != CBM_PY_ROOT_ANNOTATION_IMPORT) @@ -309,6 +331,82 @@ static py_ref_t py_ref_of_type(const cbm_resolve_binding_index_t *index, CBMAren return key ? (py_ref_t){PY_REF_KEY, key, remainder, NULL} : none; } +static py_ref_t py_ref_of_site(const cbm_resolve_binding_index_t *index, CBMArena *arena, + const cbm_py_import_lookup_t *imports, const char *chain, + uint32_t scope, uint32_t site, unsigned depth, unsigned *steps); + +/* The chain reduced through the binder every path to the site passes last. The value was read + * where that binder stands, so `x = x.clone()` follows x to the binder before it. */ +static py_ref_t py_ref_of_reach(const cbm_resolve_binding_index_t *index, CBMArena *arena, + const cbm_py_import_lookup_t *imports, const char *chain, + uint32_t scope, uint32_t site, unsigned depth, unsigned *steps) { + py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL}; + size_t length = strcspn(chain, PY_CHAIN_STOPS); + const char *rest = chain + length, *value = NULL; + uint32_t at = 0, chain_scope = scope; + bool typed = false; + cbm_resolve_python_flow_t flow = cbm_resolve_binding_python_reaching( + index, chain, length, scope, site, &value, &at, &chain_scope, &typed); + if (flow == CBM_PY_FLOW_NONE) + return py_ref_of_chain(index, arena, imports, chain, scope, 0, depth, steps); + if (flow == CBM_PY_FLOW_UNKNOWN || depth > PY_MAX_DEPTH || ++*steps > PY_MAX_REF_STEPS) + return none; + if (typed) + return py_ref_of_type(index, arena, imports, value, rest, chain_scope, depth + 1, steps); + const char *joined = cbm_arena_sprintf(arena, "%s%s", value, rest); + return joined ? py_ref_of_site(index, arena, imports, joined, chain_scope, at, depth + 1, steps) + : none; +} + +/* What a store through a prefix of the chain leaves there, held against what that prefix is + * declared to hold. Reading on is sure only where the two agree. */ +static py_ref_t py_ref_past_store(const cbm_resolve_binding_index_t *index, CBMArena *arena, + const cbm_py_import_lookup_t *imports, const char *chain, + size_t prefix, const char *value, uint32_t value_at, + uint32_t value_scope, uint32_t scope, uint32_t site, + unsigned depth, unsigned *steps) { + py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL}; + if (!value || depth > PY_MAX_DEPTH || ++*steps > PY_MAX_REF_STEPS) + return none; + py_join_t *join = cbm_arena_calloc(arena, sizeof(*join)); + py_ref_t *arms = cbm_arena_calloc(arena, 2 * sizeof(*arms)); + const char **tails = cbm_arena_calloc(arena, sizeof(*tails)); + const char *declared = cbm_arena_strndup(arena, chain, prefix); + if (!join || !arms || !tails || !declared) + return none; + arms[0] = py_ref_of_reach(index, arena, imports, declared, scope, site, depth + 1, steps); + arms[1] = py_ref_of_site(index, arena, imports, value, value_scope, value_at, depth + 1, steps); + if (arms[0].kind == PY_REF_NONE || arms[1].kind == PY_REF_NONE) + return none; + join->arms = arms; + join->arm_count = 2; + join->self_tails = tails; + return (py_ref_t){.kind = PY_REF_JOIN, .join = join, .tail = chain + prefix}; +} + +/* The chain at this site: what a narrowing says it is, what a store left there, or its binder. */ +static py_ref_t py_ref_of_site(const cbm_resolve_binding_index_t *index, CBMArena *arena, + const cbm_py_import_lookup_t *imports, const char *chain, + uint32_t scope, uint32_t site, unsigned depth, unsigned *steps) { + /* `if isinstance(x, C):` says what x is here, while isinstance is the builtin. */ + size_t consumed = 0; + const char *narrowed = + cbm_resolve_binding_python_narrowed(index, chain, scope, site, &consumed); + if (narrowed && + cbm_resolve_binding_python_root(index, "isinstance", strlen("isinstance"), scope, 0).kind == + CBM_PY_ROOT_UNBOUND) + return py_ref_of_type(index, arena, imports, narrowed, chain + consumed, scope, depth + 1, + steps); + size_t prefix = 0; + const char *stored = NULL; + uint32_t stored_at = 0, stored_scope = scope; + if (cbm_resolve_binding_python_store_precedes(index, chain, scope, site, &prefix, &stored, + &stored_at, &stored_scope)) + return py_ref_past_store(index, arena, imports, chain, prefix, stored, stored_at, + stored_scope, scope, site, depth, steps); + return py_ref_of_reach(index, arena, imports, chain, scope, site, depth, steps); +} + static py_ref_t py_keep_ref(cbm_py_members_t *members, py_ref_t ref) { ref.name = py_keep(members, ref.name); ref.tail = py_keep(members, ref.tail ? ref.tail : ""); @@ -400,8 +498,9 @@ static py_class_t *py_class_for_scope(py_class_t **by_scope, size_t count, uint3 return scope < count ? by_scope[scope] : NULL; } -/* Sort a class's members and collapse duplicates: a getter with its setter stays - * one property; any other repetition is not one provable declaration. */ +/* Sort a class's members and collapse duplicates: a getter with its setter stays one property, + * a def with `go = classmethod(go)` stays that def; any other repetition is not one provable + * declaration. */ static void py_finish_members(py_class_t *entry) { if (entry->member_count > 1) qsort(entry->members, entry->member_count, sizeof(*entry->members), py_compare_member); @@ -410,8 +509,13 @@ static void py_finish_members(py_class_t *entry) { py_member_t *member = &entry->members[i]; if (used && !strcmp(entry->members[used - 1].name, member->name)) { py_member_t *prior = &entry->members[used - 1]; - if (prior->kind != PY_MEMBER_PROPERTY || member->kind != PY_MEMBER_PROPERTY) - prior->kind = PY_MEMBER_OPAQUE; + if (prior->spelled_decorator && member->kind == PY_MEMBER_FUNCTION) + *prior = *member; + else if (!member->spelled_decorator || prior->kind != PY_MEMBER_FUNCTION) + prior->kind = + prior->kind == PY_MEMBER_PROPERTY && member->kind == PY_MEMBER_PROPERTY + ? PY_MEMBER_PROPERTY + : PY_MEMBER_OPAQUE; continue; } entry->members[used++] = *member; @@ -438,10 +542,8 @@ static bool py_note_function(cbm_py_members_t *members, const cbm_resolve_bindin return false; function->result = py_capture_type(members, index, scratch, imports, member->return_chain, member->chain_scope); - function->shape = member->shape; - function->delegate_applied = member->delegate_applied; - function->delegate = - py_capture(members, index, scratch, imports, member->delegate_chain, member->chain_scope); + function->discards = member->discards; + function->replaces = member->replaces; uint32_t count = 0; while (member->returned && member->returned[count]) count++; @@ -549,6 +651,7 @@ bool cbm_py_members_add_file(cbm_py_members_t *members, const cbm_resolve_bindin py_attribute_t *attribute = &entry->attributes[entry->attribute_count++]; attribute->name = py_keep(members, member->name); attribute->none = member->none_value; + attribute->annotated = member->type_chain != NULL; attribute->value = member->type_chain ? py_capture_type(members, index, &scratch, imports, member->type_chain, member->chain_scope) @@ -561,6 +664,11 @@ bool cbm_py_members_add_file(cbm_py_members_t *members, const cbm_resolve_bindin slot->target = py_keep(members, member->target); slot->kind = py_member_kind(index, member); slot->none_default = member->kind == CBM_PY_MEMBER_VALUE && member->none_value; + slot->spelled_decorator = member->spelled_decorator && + cbm_resolve_binding_python_root(index, member->spelled_decorator, + strlen(member->spelled_decorator), + member->owner_scope, 0) + .kind == CBM_PY_ROOT_UNBOUND; slot->annotated = member->type_chain != NULL; slot->value = member->kind == CBM_PY_MEMBER_FUNCTION ? py_capture_type(members, index, &scratch, imports, member->return_chain, @@ -574,7 +682,10 @@ bool cbm_py_members_add_file(cbm_py_members_t *members, const cbm_resolve_bindin ok = false; break; } - if (member->kind == CBM_PY_MEMBER_FUNCTION) + /* A def under a condition has no provable identity, and its returns still say whether + * a special method defers to the base. */ + if (member->kind == CBM_PY_MEMBER_FUNCTION || + (member->kind == CBM_PY_MEMBER_OPAQUE && member->returned)) ok = py_note_function(members, index, &scratch, imports, member); } for (size_t c = 0; ok && c < classes; c++) { @@ -656,7 +767,9 @@ typedef enum { PY_VALUE_INSTANCE, PY_VALUE_SUPER, PY_VALUE_FUNCTION, - PY_VALUE_EXTERNAL + PY_VALUE_EXTERNAL, + PY_VALUE_SEQ, /* holds and yields instances of cls */ + PY_VALUE_MAP /* holds instances of cls by key; what it yields is the keys */ } py_value_kind_t; typedef struct { @@ -666,6 +779,9 @@ typedef struct { const py_ref_t *result; /* annotated return of a function */ const char *declaration; /* the member read last, for a reference to it */ const char *owner; + const py_class_t *receiver; /* the class a method was read through */ + /* An instance, or class, standing for whatever self or cls is at run time. */ + bool of_receiver; } py_value_t; typedef struct { @@ -747,14 +863,46 @@ static const py_member_t *py_lookup(const py_class_t *cls, const char *name, uin return NULL; } -/* What instances store under this name, across the certain linearization. All +/* What instances store under this name: the nearest class in the certain linearization that + * stores it, up to the class that declares the member when one does. A store further along is + * another declaration of the name, and the nearer one selects, as it does for a method. A class's * typed stores must agree; a None store says nothing about the class. */ static bool py_instance_attribute(const py_eval_t *eval, const py_class_t *cls, const char *name, - py_value_t *out, unsigned depth) { + const py_class_t *declarer, py_value_t *out, unsigned depth) { bool present = false, unknown = false; py_value_t agreed = {.kind = PY_VALUE_UNKNOWN}; + /* An annotation is the attribute's declared type wherever it stands, and what a subclass + * stores without one is held to it. */ + const py_attribute_t *declared = NULL; + for (uint32_t i = 0; i < cls->reliable && !declared; i++) { + const py_class_t *owner = cls->mro[i]; + if (i && declarer && cls->mro[i - 1] == declarer) + break; + py_attribute_t key = {.name = name}; + const py_attribute_t *found = + owner->attribute_count ? bsearch(&key, owner->attributes, owner->attribute_count, + sizeof(*owner->attributes), py_compare_attribute) + : NULL; + if (!found) + continue; + while (found > owner->attributes && !strcmp(found[-1].name, name)) + found--; + for (; found < owner->attributes + owner->attribute_count && !strcmp(found->name, name); + found++) + if (found->annotated) { + declared = found; + break; + } + } + if (declared) { + *out = py_eval_ref(eval, &declared->value, depth + 1); + return true; + } for (uint32_t i = 0; i < cls->reliable && !unknown; i++) { const py_class_t *owner = cls->mro[i]; + if (i && declarer && cls->mro[i - 1] == declarer) + break; + py_attribute_t key = {.name = name}; const py_attribute_t *found = owner->attribute_count ? bsearch(&key, owner->attributes, owner->attribute_count, @@ -770,46 +918,60 @@ static bool py_instance_attribute(const py_eval_t *eval, const py_class_t *cls, if (found->none) continue; py_value_t value = py_eval_ref(eval, &found->value, depth + 1); - if (value.kind != PY_VALUE_INSTANCE || - (agreed.kind != PY_VALUE_UNKNOWN && agreed.cls != value.cls)) { + bool holds = value.kind == PY_VALUE_INSTANCE || value.kind == PY_VALUE_SEQ || + value.kind == PY_VALUE_MAP; + if (!holds || (agreed.kind != PY_VALUE_UNKNOWN && + (agreed.cls != value.cls || agreed.kind != value.kind))) { unknown = true; break; } agreed = value; } + if (agreed.kind != PY_VALUE_UNKNOWN) + break; } *out = unknown ? (py_value_t){.kind = PY_VALUE_UNKNOWN} : agreed; return present; } -/* A metaclass or __init_subclass__ can replace what a class body assigned: a Django field - * reads as its column value, an Enum member as the enum. A base we cannot resolve at all - * could be anything. An external class is taken to keep it, as unittest.TestCase keeps - * rf = RequestFactory(); the known exceptions from the stdlib count as a metaclass. */ +/* What a class body assigned is the attribute's declared value, and a read selects that: + * what a metaclass does to it is not a static fact, and a method called on such a read has to + * exist on the declared class to resolve at all. Enum and NamedTuple are the exceptions the + * standard library states: a member reads as the enum. A base we cannot resolve at all could + * be anything. */ static bool py_class_body_rewritable(const py_class_t *cls) { if (cls->uncertain) return true; - for (uint32_t i = 0; i < cls->mro_count; i++) { - const py_class_t *entry = cls->mro[i]; - if (entry->metaclass || py_own_member(entry, "__init_subclass__")) + for (uint32_t i = 0; i < cls->mro_count; i++) + if (cls->mro[i]->metaclass) return true; - } return false; } +static py_value_t py_function_returns(const py_eval_t *eval, const char *qualified_name, + const py_class_t *receiver, bool of_receiver, unsigned depth); + +static py_value_t py_special_step(const py_eval_t *eval, const py_value_t *value, const char *name, + unsigned depth); + static py_value_t py_read_member(const py_eval_t *eval, const py_value_t *value, const char *name, unsigned depth) { py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; const py_class_t *cls = value->cls; if (value->kind == PY_VALUE_INSTANCE && !strcmp(name, "__class__")) - return (py_value_t){.kind = PY_VALUE_CLASS, .cls = cls}; + return (py_value_t){.kind = PY_VALUE_CLASS, .cls = cls, .of_receiver = value->of_receiver}; if (value->kind == PY_VALUE_CLASS && !strcmp(name, "")) return (py_value_t){.kind = PY_VALUE_SUPER, .cls = cls}; const py_class_t *owner = NULL; const py_member_t *member = py_lookup(cls, name, value->kind == PY_VALUE_SUPER ? 1 : 0, &owner); - if (value->kind == PY_VALUE_INSTANCE) { + /* `cls.user = User()` in a classmethod is read back through the class, and only code that + * stored it that way can read it so. What the class declares is what its class answers: + * `self.go = make()` says nothing about `Child.go`. */ + if (value->kind == PY_VALUE_INSTANCE || + (value->kind == PY_VALUE_CLASS && (!member || member->none_default))) { py_value_t stored; - if (py_instance_attribute(eval, cls, name, &stored, depth)) { + + if (py_instance_attribute(eval, cls, name, member ? owner : NULL, &stored, depth)) { if (member && member->kind != PY_MEMBER_PROPERTY && !member->none_default) return unknown; if (!member || member->none_default) @@ -828,17 +990,36 @@ static py_value_t py_read_member(const py_eval_t *eval, const py_value_t *value, .qualified_name = member->target, .result = &member->value, .declaration = member->target, - .owner = owner->qualified_name}; + .owner = owner->qualified_name, + .receiver = cls, + .of_receiver = value->of_receiver}; case PY_MEMBER_PROPERTY: { - py_value_t read = value->kind == PY_VALUE_INSTANCE || value->kind == PY_VALUE_SUPER - ? py_eval_ref(eval, &member->value, depth + 1) - : unknown; + bool instance = value->kind == PY_VALUE_INSTANCE || value->kind == PY_VALUE_SUPER; + py_value_t read = + !instance ? unknown + : member->value.kind != PY_REF_NONE + + ? py_eval_ref(eval, &member->value, depth + 1) + : py_function_returns(eval, member->target, cls, value->of_receiver, depth + 1); read.declaration = member->target; read.owner = owner->qualified_name; return read; } case PY_MEMBER_VALUE: { + /* A class declared in this class's body. */ + if (member->value.kind == PY_REF_NONE) { + const py_class_t *nested = + py_class(eval->members, + cbm_arena_sprintf(eval->arena, "%s.%s", owner->qualified_name, name)); + if (nested) + return (py_value_t){.kind = PY_VALUE_CLASS, .cls = nested}; + } py_value_t read = py_eval_ref(eval, &member->value, depth + 1); + /* A class-body value whose class has __get__ is a descriptor: a read runs that. */ + const py_class_t *getter = NULL; + if (read.kind == PY_VALUE_INSTANCE && !member->annotated && + py_lookup(read.cls, "__get__", 0, &getter)) + return py_special_step(eval, &read, "__get__", depth + 1); /* An annotation states what a read yields, whatever the machinery does. */ return read.kind == PY_VALUE_INSTANCE && !member->annotated && py_class_body_rewritable(cls) ? unknown @@ -858,32 +1039,55 @@ static py_value_t py_read_member(const py_eval_t *eval, const py_value_t *value, static void py_infer_returns(const py_eval_t *eval, py_function_t *function, unsigned depth) { function->inference = 1; const py_class_t *agreed = NULL; - bool known = function->returned_count > 0 && depth <= PY_MAX_DEPTH; + bool known = function->returned_count > 0 && depth <= PY_MAX_DEPTH, receiver = known; + py_value_kind_t kind = PY_VALUE_UNKNOWN; for (uint32_t i = 0; known && i < function->returned_count; i++) { py_value_t value = py_eval_ref(eval, &function->returned[i], depth + 1); - known = value.kind == PY_VALUE_INSTANCE && (!agreed || agreed == value.cls); + known = (value.kind == PY_VALUE_INSTANCE || value.kind == PY_VALUE_CLASS) && + (!agreed || (agreed == value.cls && kind == value.kind)); agreed = value.cls; + kind = value.kind; + receiver = receiver && value.of_receiver; } function->inferred = known ? agreed : NULL; + function->returns_class = known && kind == PY_VALUE_CLASS; + function->returns_receiver = known && receiver; function->inference = 2; } +/* What an unannotated function returns. `return self` in an inherited method is the class it + * was read through, as a type checker reads an unannotated self: chord(...).set() is a chord + * although set is declared on Signature. */ +static py_value_t py_function_returns(const py_eval_t *eval, const char *qualified_name, + const py_class_t *receiver, bool of_receiver, + unsigned depth) { + py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; + py_function_t *function = cbm_ht_get(eval->members->functions, qualified_name); + if (function && function->inference == 0) + py_infer_returns(eval, function, depth); + if (!function || !function->inferred) + return unknown; + + return (py_value_t){.kind = function->returns_class ? PY_VALUE_CLASS : PY_VALUE_INSTANCE, + .cls = + function->returns_receiver && receiver ? receiver : function->inferred, + .of_receiver = function->returns_receiver && of_receiver}; +} + static py_value_t py_call(const py_eval_t *eval, const py_value_t *value, unsigned depth) { py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; switch (value->kind) { case PY_VALUE_CLASS: /* __new__ can return anything. */ return value->cls->custom_new ? unknown - : (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = value->cls}; + : (py_value_t){.kind = PY_VALUE_INSTANCE, + .cls = value->cls, + .of_receiver = value->of_receiver}; case PY_VALUE_FUNCTION: { if (value->result && value->result->kind != PY_REF_NONE) return py_eval_ref(eval, value->result, depth + 1); - py_function_t *function = cbm_ht_get(eval->members->functions, value->qualified_name); - if (function && function->inference == 0) - py_infer_returns(eval, function, depth); - return function && function->inferred - ? (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = function->inferred} - : unknown; + return py_function_returns(eval, value->qualified_name, value->receiver, value->of_receiver, + depth); } case PY_VALUE_INSTANCE: { const py_class_t *owner = NULL; @@ -896,6 +1100,79 @@ static py_value_t py_call(const py_eval_t *eval, const py_value_t *value, unsign } } +/* A special method of a project class answering a container step. */ +static py_value_t py_special_step(const py_eval_t *eval, const py_value_t *value, const char *name, + unsigned depth) { + py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; + if (value->kind != PY_VALUE_INSTANCE) + return unknown; + const py_class_t *owner = NULL; + const py_member_t *member = py_lookup(value->cls, name, 0, &owner); + if (!member || member->kind != PY_MEMBER_FUNCTION) + return unknown; + py_value_t bound = {.kind = PY_VALUE_FUNCTION, + .qualified_name = member->target, + .result = &member->value, + .receiver = value->cls, + .of_receiver = value->of_receiver}; + return py_call(eval, &bound, depth); +} + +/* x[k], x[a:b], and iterating or wrapping x, as the annotation said. */ +static py_value_t py_container_step(const py_eval_t *eval, const py_value_t *value, + const char *step, size_t length, unsigned depth) { + py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; + if (value->kind == PY_VALUE_EXTERNAL) + return *value; + bool seq = value->kind == PY_VALUE_SEQ, map = value->kind == PY_VALUE_MAP; + py_value_t element = {.kind = PY_VALUE_INSTANCE, .cls = value->cls}; + if (length == 4 && !strncmp(step, "@seq", 4)) { + return value->kind == PY_VALUE_INSTANCE + ? (py_value_t){.kind = PY_VALUE_SEQ, .cls = value->cls} + : unknown; + } + if (length == 4 && !strncmp(step, "@map", 4)) { + return value->kind == PY_VALUE_INSTANCE + ? (py_value_t){.kind = PY_VALUE_MAP, .cls = value->cls} + : unknown; + } + /* d.items() against an annotated map: a class's own items() means what it says. */ + if (length == 7 && !strncmp(step, "@values", 7)) + return map ? (py_value_t){.kind = PY_VALUE_SEQ, .cls = value->cls} : unknown; + if (length == 5 && !strncmp(step, "@iter", 5)) { + if (seq) + return element; + if (map) + return unknown; + py_value_t iterator = py_special_step(eval, value, "__iter__", depth); + return iterator.kind == PY_VALUE_SEQ + ? (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = iterator.cls} + : unknown; + } + if (length == 2 && !strncmp(step, "[]", 2)) { + if (seq || map) + return element; + return py_special_step(eval, value, "__getitem__", depth); + } + if (length == 3 && !strncmp(step, "[:]", 3)) + return seq ? *value : py_special_step(eval, value, "__getitem__", depth); + return unknown; +} + +/* The container methods that hand back what it holds. */ +static py_value_t py_container_method(const py_value_t *value, const char *name) { + py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; + py_value_t element = {.kind = PY_VALUE_INSTANCE, .cls = value->cls}; + if (!strcmp(name, "copy")) + return *value; + if (value->kind == PY_VALUE_SEQ) + return !strcmp(name, "pop") ? element : unknown; + if (!strcmp(name, "get") || !strcmp(name, "pop") || !strcmp(name, "setdefault")) + return element; + return !strcmp(name, "values") ? (py_value_t){.kind = PY_VALUE_SEQ, .cls = value->cls} + : unknown; +} + static py_value_t py_apply_tail(const py_eval_t *eval, py_value_t value, const char *tail, unsigned depth) { py_value_t unknown = {.kind = PY_VALUE_UNKNOWN}; @@ -905,14 +1182,33 @@ static py_value_t py_apply_tail(const py_eval_t *eval, py_value_t value, const c tail += 2; continue; } + if (tail[0] == '@' || tail[0] == '[') { + size_t step = + tail[0] == '[' ? strcspn(tail, "]") + 1 : 1 + strcspn(tail + 1, PY_CHAIN_STOPS); + value = py_container_step(eval, &value, tail, step, depth); + tail += step; + continue; + } if (tail[0] != '.') return unknown; tail++; - size_t length = strcspn(tail, ".("); + size_t length = strcspn(tail, PY_CHAIN_STOPS); const char *name = cbm_arena_strndup(eval->arena, tail, length); if (!name) return unknown; tail += length; + /* A with statement on what a contextmanager generator yields. */ + if (value.kind == PY_VALUE_SEQ && + (!strcmp(name, "__enter__") || !strcmp(name, "__aenter__")) && tail[0] == '(') { + value = (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = value.cls}; + tail += 2; + continue; + } + if ((value.kind == PY_VALUE_SEQ || value.kind == PY_VALUE_MAP) && tail[0] == '(') { + value = py_container_method(&value, name); + tail += 2; + continue; + } switch (value.kind) { case PY_VALUE_MODULE: { const char *target = NULL; @@ -964,7 +1260,8 @@ static py_value_t py_eval_ref(const py_eval_t *eval, const py_ref_t *ref, unsign value = (py_value_t){.kind = ref->kind == PY_REF_SELF ? PY_VALUE_INSTANCE : ref->kind == PY_REF_CLASS ? PY_VALUE_CLASS : PY_VALUE_SUPER, - .cls = cls}; + .cls = cls, + .of_receiver = true}; break; } case PY_REF_JOIN: { @@ -994,75 +1291,36 @@ static py_value_t py_eval_ref(const py_eval_t *eval, const py_ref_t *ref, unsign typedef enum { PY_KEEP_NONE = 0, PY_KEEP_IDENTITY, PY_KEEP_PROPERTY } py_keep_t; -/* Standard library decorators whose result is the function or class itself, or a - * thin wrapper that calls it. Facts about those modules, not about this project. */ -static bool py_stdlib_keeps_identity(const char *canonical, bool applied) { - static const char *const direct[] = {"abc.abstractmethod", - "typing.final", - "typing.override", - "typing.no_type_check", - "typing_extensions.final", - "typing_extensions.override", - "functools.lru_cache", - "functools.cache", - "functools.total_ordering", - "contextlib.contextmanager", - "contextlib.asynccontextmanager", - "dataclasses.dataclass", - "enum.unique", - "typing.runtime_checkable", - "atexit.register", - "typing_extensions.runtime_checkable", - NULL}; - static const char *const called[] = {"functools.lru_cache", "functools.wraps", - "dataclasses.dataclass", NULL}; - for (const char *const *name = applied ? called : direct; *name; name++) - if (!strcmp(*name, canonical)) - return true; - return false; -} - -static bool py_stdlib_property(const char *canonical) { - return !strcmp(canonical, "functools.cached_property") || - !strcmp(canonical, "abc.abstractproperty"); -} - -/* The source proves it keeps identity in this form, or hands the function on to a - * decorator that does. */ -static bool py_function_keeps(const py_eval_t *eval, const char *qualified_name, bool applied, - unsigned depth) { - const py_function_t *function = cbm_ht_get(eval->members->functions, qualified_name); - if (!function || depth > 4) - return false; - if (function->shape & (applied ? CBM_PY_SHAPE_FACTORY : CBM_PY_SHAPE_DIRECT)) - return true; - if (applied || !(function->shape & CBM_PY_SHAPE_DELEGATES)) - return false; - py_value_t next = py_eval_ref(eval, &function->delegate, depth + 1); - return next.kind == PY_VALUE_FUNCTION && - py_function_keeps(eval, next.qualified_name, function->delegate_applied, depth + 1); +/* functools.cached_property, kombu's, werkzeug's, abc.abstractproperty: reading such a member + * runs the function, and what a site then calls is its value. Saying so only ever withholds + * a call. */ +static bool py_external_property(const char *canonical) { + const char *dot = strrchr(canonical, '.'); + const char *name = dot ? dot + 1 : canonical; + size_t length = strlen(name); + return length >= 8 && !strcmp(name + length - 8, "property"); } +/* A decorator does not change which declaration a name selects. What runs when the name is + * called can differ, as it can behind any dispatch, and CALLS is not that. So every decorator + * keeps the declaration, the way a type checker reads one it cannot see through, unless the + * source proves it discards the function. */ static py_keep_t py_decorator_verdict(const py_eval_t *eval, const py_decorator_t *decorator) { py_ref_t bare = decorator->ref; - if (bare.kind == PY_REF_UNBOUND && (!bare.tail || !bare.tail[0])) { - if (decorator->applied) - return PY_KEEP_NONE; - if (!strcmp(bare.name, "staticmethod") || !strcmp(bare.name, "classmethod")) - return PY_KEEP_IDENTITY; - return !strcmp(bare.name, "property") ? PY_KEEP_PROPERTY : PY_KEEP_NONE; - } + if (bare.kind == PY_REF_UNBOUND && (!bare.tail || !bare.tail[0])) + return !decorator->applied && !strcmp(bare.name, "property") ? PY_KEEP_PROPERTY + : PY_KEEP_IDENTITY; py_value_t value = py_eval_ref(eval, &bare, 0); - if (value.kind == PY_VALUE_EXTERNAL && decorator->canonical) { - if (py_stdlib_keeps_identity(decorator->canonical, decorator->applied)) - return PY_KEEP_IDENTITY; - return !decorator->applied && py_stdlib_property(decorator->canonical) ? PY_KEEP_PROPERTY - : PY_KEEP_NONE; + + if (value.kind == PY_VALUE_EXTERNAL && decorator->canonical) + return !decorator->applied && py_external_property(decorator->canonical) ? PY_KEEP_PROPERTY + : PY_KEEP_IDENTITY; + if (value.kind == PY_VALUE_FUNCTION) { + const py_function_t *function = cbm_ht_get(eval->members->functions, value.qualified_name); + return function && (function->discards || (function->replaces && !decorator->applied)) + ? PY_KEEP_NONE + : PY_KEEP_IDENTITY; } - if (value.kind == PY_VALUE_FUNCTION) - return py_function_keeps(eval, value.qualified_name, decorator->applied, 0) - ? PY_KEEP_IDENTITY - : PY_KEEP_NONE; /* A classmethod or staticmethod subclass binds the same function. */ for (uint32_t i = 0; value.kind == PY_VALUE_CLASS && !decorator->applied && i < value.cls->base_count; i++) { @@ -1074,7 +1332,7 @@ static py_keep_t py_decorator_verdict(const py_eval_t *eval, const py_decorator_ /* A descriptor class: `@cached_property` reads run the function once. */ if (value.kind == PY_VALUE_CLASS && !decorator->applied && py_own_member(value.cls, "__get__")) return PY_KEEP_PROPERTY; - return PY_KEEP_NONE; + return PY_KEEP_IDENTITY; } bool cbm_py_members_decide_decorated(cbm_py_members_t *members, const cbm_py_members_env_t *env, @@ -1306,11 +1564,36 @@ static uint32_t py_single_chain(const py_class_t *cls, const py_class_t **mro, u return reliable; } -/* Base[int] asks the class, and a __class_getitem__ can answer with anything. */ -static bool py_answers_subscript(const py_class_t *cls) { - for (uint32_t i = 0; i < cls->mro_count; i++) - if (py_own_member(cls->mro[i], "__class_getitem__")) +/* This special method answers through the base on some path: `return super().__new__(cls)`, + * or django's __getattribute__ reading `super().__getattribute__(name)`. Then the declared + * members are still what a read finds, and a call of the class still makes an instance. */ +static bool py_defers_to_base(const cbm_py_members_t *members, const py_member_t *member, + const char *tail) { + const py_function_t *function = cbm_ht_get(members->functions, member->target); + size_t tail_length = strlen(tail); + + for (uint32_t i = 0; function && i < function->returned_count; i++) { + const py_ref_t *returned = &function->returned[i]; + size_t length = returned->tail ? strlen(returned->tail) : 0; + bool base = returned->kind == PY_REF_SUPER || + (returned->kind == PY_REF_UNBOUND && !strcmp(returned->name, "object")) || + (length > tail_length && + !strncmp(returned->tail + length - tail_length - strlen("."), + ".", strlen("."))); + if (base && length >= tail_length && !strcmp(returned->tail + length - tail_length, tail)) return true; + } + return false; +} + +/* Base[int] asks the class, and a __class_getitem__ can answer with anything. click's fills + * in a parameter and answers through super, so Base[int] is still Base. */ +static bool py_answers_subscript(const cbm_py_members_t *members, const py_class_t *cls) { + for (uint32_t i = 0; i < cls->mro_count; i++) { + const py_member_t *member = py_own_member(cls->mro[i], "__class_getitem__"); + if (member && !py_defers_to_base(members, member, ".__class_getitem__()")) + return true; + } return false; } @@ -1346,7 +1629,7 @@ static void py_seal_class(const py_eval_t *eval, cbm_py_members_t *members, py_c } py_seal_class(eval, members, (py_class_t *)base, depth + 1); if (!base->sealed || - (b < 64 && ((cls->subscripted >> b) & 1) && py_answers_subscript(base))) { + (b < 64 && ((cls->subscripted >> b) & 1) && py_answers_subscript(members, base))) { unsealed = true; continue; } @@ -1382,8 +1665,10 @@ static void py_seal_class(const py_eval_t *eval, cbm_py_members_t *members, py_c reliable = py_single_chain(cls, mro, mro_count); cls->reliable = reliable; for (uint32_t i = 0; i < reliable; i++) { - cls->intercepts |= py_own_member(mro[i], "__getattribute__") != NULL; - cls->custom_new |= py_own_member(mro[i], "__new__") != NULL; + const py_member_t *read = py_own_member(mro[i], "__getattribute__"); + const py_member_t *made = py_own_member(mro[i], "__new__"); + cls->intercepts |= read && !py_defers_to_base(members, read, ".__getattribute__()"); + cls->custom_new |= made && !py_defers_to_base(members, made, ".__new__()"); } cls->visiting = false; cls->sealed = true; @@ -1495,7 +1780,7 @@ bool cbm_py_members_resolve(const cbm_py_members_t *members, const cbm_py_member !cbm_resolve_binding_python_site(index, name, start, end, origin, &chain, &scope)) return false; unsigned ref_steps = 0, eval_steps = 0; - py_ref_t ref = py_ref_of_chain(index, arena, imports, chain, scope, 0, 0, &ref_steps); + py_ref_t ref = py_ref_of_site(index, arena, imports, chain, scope, start, 0, &ref_steps); if (ref.kind == PY_REF_NONE || ref.kind == PY_REF_UNBOUND) return false; py_eval_t eval = {.members = members, .env = env, .arena = arena, .steps = &eval_steps}; diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index 9038112be..696d143e7 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -15,6 +15,8 @@ #include "pipeline/pass_lsp_cross.h" #include "pipeline/import_resolve.h" #include "pipeline/resolver_ecma.h" +#include "pipeline/py_layout.h" + #include "resolve/capabilities.h" #include "resolve/policy.h" #include "resolve/rust_names.h" @@ -1251,6 +1253,77 @@ static void parse_package_swift(const char *source, int source_len, const char * swift_scan_membership_only(source, end, rel_path, ".executableTarget(", "Sources", entries); } +/* Python: the source roots a manifest states, one repository directory per line. The + * conventional `src` beside a manifest is a root for every build backend, as a flat + * layout is its own directory. A root that holds no such module answers nothing. */ +typedef struct { + const char *directory; + char roots[PKGMAP_PATH_BUF]; + size_t used; + cbm_pkg_entries_t *entries; +} python_roots_t; + +static void python_add_root(void *context, const char *root, size_t length) { + python_roots_t *roots = context; + char written[PKGMAP_PATH_BUF], path[PKGMAP_PATH_BUF]; + if (length >= sizeof(written)) + return; + memcpy(written, root, length); + written[length] = '\0'; + if (strpbrk(written, "*?\\\n") || written[0] == '/' || + !cbm_repo_rel_join(roots->directory, written, path, sizeof(path)) || !path[0]) + return; + size_t size = strlen(path); + if (roots->used + size + 2 > sizeof(roots->roots)) + return; + /* Only a root apart from the manifest makes everything below it a module: a file + * in a flat project's own directory can still be a script. */ + if (strcmp(path, roots->directory) != 0) + node_collect(roots->entries, "py-root", path, NULL, "1"); + memcpy(roots->roots + roots->used, path, size); + roots->used += size; + roots->roots[roots->used++] = '\n'; + roots->roots[roots->used] = '\0'; +} + +typedef struct { + char text[PKGMAP_PATH_BUF]; + size_t used, count; +} python_members_t; + +static void python_add_member(void *context, const char *member, size_t length) { + python_members_t *members = context; + if (!length || memchr(member, '\n', length) || + members->used + length + 2 > sizeof(members->text)) + return; + memcpy(members->text + members->used, member, length); + members->used += length; + members->text[members->used++] = '\n'; + members->text[members->used] = '\0'; + members->count++; +} + +static void parse_python_manifest(const char *basename, const char *source, int source_len, + const char *rel_path, cbm_pkg_entries_t *entries) { + char *directory = path_dirname(rel_path); + if (!directory) { + entries->incomplete = true; + return; + } + python_roots_t roots = {.directory = directory, .entries = entries}; + python_add_root(&roots, "src", 3); + python_add_root(&roots, ".", 1); + cbm_py_layout_parse(basename, source, (size_t)source_len, python_add_root, &roots); + node_collect(entries, "py-project", directory, basename, roots.roots); + if (!strcmp(basename, "pyproject.toml")) { + python_members_t members = {.count = 0}; + cbm_py_layout_workspace_members(source, (size_t)source_len, python_add_member, &members); + if (members.count) + node_collect(entries, "py-workspace", directory, NULL, members.text); + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, directory); +} + /* ── Public: manifest detection + parsing ──────────────────────── */ bool cbm_pkgmap_try_parse(const char *basename, const char *rel_path, const char *source, @@ -1273,11 +1346,11 @@ bool cbm_pkgmap_try_parse(const char *basename, const char *rel_path, const char parse_cargo_toml(source, source_len, rel_path, entries); return true; } - if (strcmp(basename, "pyproject.toml") == 0) { + if (cbm_py_layout_is_manifest(basename)) { /* A distribution name is not an import path or a package directory. - * Do not invent competing root/src mappings from [project].name. - * Exact physical modules and explicitly supplied mappings retain - * their existing independent resolution paths. */ + * Do not invent competing root/src mappings from [project].name: the + * manifest's layout is what says where modules live. */ + parse_python_manifest(basename, source, source_len, rel_path, entries); return true; } if (strcmp(basename, "composer.json") == 0) { @@ -1420,6 +1493,12 @@ static void node_parent_path(char *path) { path[0] = '\0'; } +/* The directory of a private fact of this kind, without reading the map. */ +static bool python_fact_key(const cbm_pkg_entry_t *item, const char *kind, char *directory, + size_t capacity) { + return item->node_fact && node_fact_directory(item->pkg_name, kind, directory, capacity); +} + static bool node_glob_covered(CBMHashTable *map, const char *directory) { if (!node_fact(map, "directory", directory, NULL) || node_fact(map, "excluded-child", directory, NULL)) @@ -1709,16 +1788,19 @@ static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_ent excluded_count); continue; } - if (!is_pkgmap_manifest_basename(name)) + /* setup.cfg and setup.py only ever add a source root, so one that cannot + * be read withholds that and nothing else. */ + bool optional = cbm_py_layout_is_manifest(name) && !is_pkgmap_manifest_basename(name); + if (!is_pkgmap_manifest_basename(name) && !optional) continue; if (!info.is_regular) { - entries->incomplete = true; + entries->incomplete = entries->incomplete || !optional; continue; } int source_len = 0; char *source = pkgmap_read_file(abs_path, &source_len); if (!source) { - entries->incomplete = true; + entries->incomplete = entries->incomplete || !optional; continue; } if (cbm_pkgmap_try_parse(name, rel_path, source, source_len, entries)) { @@ -1743,12 +1825,97 @@ static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_ent * coverage, including symlinks / Windows reparse points inside the domain. * This is what lets bare workspace imports (e.g. "@org/pkg" declared in * an ignored package.json) resolve on Windows as well as POSIX. */ +/* One member root's top-level names, under the workspace that lists it. Two members providing a + * name collide in the map's merge, which leaves that name to no one. */ +static void python_note_member_root(const char *repo_path, const char *workspace, const char *root, + cbm_pkg_entries_t *entries) { + char absolute[PKGMAP_PATH_BUF]; + int size = snprintf(absolute, sizeof(absolute), "%s%s%s", repo_path, root[0] ? "/" : "", root); + cbm_dir_t *dir = size > 0 && (size_t)size < sizeof(absolute) ? cbm_opendir(absolute) : NULL; + if (!dir) + return; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + char name[PKGMAP_PATH_BUF]; + int length = snprintf(name, sizeof(name), "%s", entry->name); + if (length <= 0 || (size_t)length >= sizeof(name) || entry->name[0] == '.') + continue; + if (length > 3 && !strcmp(name + length - 3, ".py")) + name[length - 3] = '\0'; + if (!strchr(name, '.')) + node_collect(entries, "py-member-head", workspace, name, root); + } + cbm_closedir(dir); +} + +/* One project under one ancestor that declares a workspace, when that workspace lists it. */ +static void python_note_member(const char *repo_path, const char *workspace, const char *project, + const char *patterns, const char *roots, + cbm_pkg_entries_t *entries) { + size_t prefix = strlen(workspace); + const char *below = prefix ? project + prefix + 1 : project; + bool member = false; + for (const char *pattern = patterns; *pattern && !member;) { + size_t length = strcspn(pattern, "\n"); + char one[PKGMAP_PATH_BUF]; + if (length < sizeof(one)) { + memcpy(one, pattern, length); + one[length] = '\0'; + member = cbm_py_layout_member_matches(one, below); + } + pattern += length + (pattern[length] != '\0'); + } + for (const char *root = roots; member && *root;) { + size_t length = strcspn(root, "\n"); + char one[PKGMAP_PATH_BUF]; + if (length < sizeof(one)) { + memcpy(one, root, length); + one[length] = '\0'; + python_note_member_root(repo_path, workspace, one, entries); + } + root += length + (root[length] != '\0'); + } +} + +/* A uv workspace's members are installed together. Each member project's roots are read once, + * after the walk has seen every manifest: one pass over the collected facts, one lookup per + * ancestor of each project, plus the size of the member directories. */ +static void python_finish_workspaces(const char *repo_path, cbm_pkg_entries_t *entries) { + int total = entries->count; + CBMHashTable *spaces = cbm_ht_create(8); + if (!spaces) + return; + for (int w = 0; w < total; w++) { + char workspace[PKGMAP_PATH_BUF]; + if (python_fact_key(&entries->items[w], "py-workspace", workspace, sizeof(workspace))) + cbm_ht_set(spaces, entries->items[w].pkg_name, entries->items[w].entry_rel); + } + for (int p = 0; p < total && cbm_ht_count(spaces); p++) { + char project[PKGMAP_PATH_BUF], ancestor[PKGMAP_PATH_BUF], key[PKGMAP_PATH_BUF]; + if (!python_fact_key(&entries->items[p], "py-project", project, sizeof(project)) || + !project[0]) + continue; + const char *roots = entries->items[p].entry_rel; + memcpy(ancestor, project, strlen(project) + 1); + do { + node_parent_path(ancestor); + const char *patterns = node_fact_key(key, sizeof(key), "py-workspace", ancestor, NULL) + ? cbm_ht_get(spaces, key) + : NULL; + if (patterns) + python_note_member(repo_path, ancestor, project, patterns, roots, entries); + } while (ancestor[0]); + } + cbm_ht_free(spaces); +} + int cbm_pkgmap_scan_repo(const char *repo_path, cbm_pkg_entries_t *entries, char **excluded_dirs, int excluded_count) { if (!repo_path || !entries) { return 0; } int parsed = pkgmap_walk_dir(repo_path, "", entries, 0, excluded_dirs, excluded_count); + python_finish_workspaces(repo_path, entries); cbm_log_info("pkgmap.scan_repo", "manifests", pkgmap_itoa(parsed)); return parsed; } @@ -3707,13 +3874,16 @@ static const cbm_gbuf_node_t *python_configured_module(const cbm_pipeline_ctx_t typedef struct { char base[PKGMAP_PATH_BUF]; /* empty at the repo root */ char own[PKGMAP_PATH_BUF]; /* the file's top-level package, empty outside one */ + /* A source root above base: the directories between are namespace packages. */ + char root[PKGMAP_PATH_BUF]; /* empty when base itself is the search directory */ + char root_own[PKGMAP_PATH_BUF]; /* the file's top-level name under that root */ } python_search_t; /* Where a file's absolute imports look first. A file inside a package was loaded under * that package, so the directory above its topmost __init__.py is on sys.path. A file * outside any package is a script or a test, and its own directory is. */ -static bool python_search_path(const cbm_pipeline_ctx_t *ctx, const char *source_rel, - python_search_t *search) { +static bool python_package_base(const cbm_pipeline_ctx_t *ctx, const char *source_rel, + python_search_t *search) { search->own[0] = '\0'; int length = snprintf(search->base, sizeof(search->base), "%s", source_rel); if (length <= 0 || (size_t)length >= sizeof(search->base)) @@ -3743,6 +3913,145 @@ static bool python_search_path(const cbm_pipeline_ctx_t *ctx, const char *source } } +/* A package without __init__.py is still one under a stated source root: poetry keeps + * src/poetry so, to share the name with poetry-core. The nearest root above the package + * base decides what the file is, however many such directories lie between. */ +static bool python_search_path(const cbm_pipeline_ctx_t *ctx, const char *source_rel, + python_search_t *search) { + search->root[0] = '\0'; + search->root_own[0] = '\0'; + if (!python_package_base(ctx, source_rel, search)) + return false; + CBMHashTable *map = cbm_pipeline_get_pkgmap(); + if (!map || !search->base[0]) + return true; + + char directory[PKGMAP_PATH_BUF]; + snprintf(directory, sizeof(directory), "%s", search->base); + if (node_fact(map, "py-root", directory, NULL)) + return true; + for (node_parent_path(directory); directory[0]; node_parent_path(directory)) { + const char *fact = node_fact(map, "py-root", directory, NULL); + if (!fact || !strcmp(fact, ambiguous_pkgmap_value)) + continue; + size_t size = strlen(directory); + const char *below = source_rel + size + 1; + int written = snprintf(search->root, sizeof(search->root), "%s", directory); + int own = snprintf(search->root_own, sizeof(search->root_own), "%.*s", + (int)strcspn(below, "/"), below); + return written > 0 && (size_t)written < sizeof(search->root) && own > 0 && + (size_t)own < sizeof(search->root_own); + } + return true; +} + +/* What an import names under one search directory. A module of its first name that + * lacks the rest hides anything further along. */ +static const cbm_gbuf_node_t *python_under(const cbm_pipeline_ctx_t *ctx, const char *directory, + const char *normalized, int head, bool *hidden) { + char near_path[PKGMAP_PATH_BUF], head_path[PKGMAP_PATH_BUF]; + int near_size = snprintf(near_path, sizeof(near_path), "%s%s%s", directory, + directory[0] ? "/" : "", normalized); + int head_size = snprintf(head_path, sizeof(head_path), "%s%s%.*s", directory, + directory[0] ? "/" : "", head, normalized); + if (near_size <= 0 || (size_t)near_size >= sizeof(near_path) || head_size <= 0 || + (size_t)head_size >= sizeof(head_path)) { + *hidden = true; + return NULL; + } + const cbm_gbuf_node_t *found = python_module_at_path(ctx, near_path); + *hidden = !found && python_module_at_path(ctx, head_path); + return found; +} + +/* A stated source root that is itself a package holds no top-level module. */ +static bool python_root_is_package(const cbm_pipeline_ctx_t *ctx, const char *root, size_t size) { + char init[PKGMAP_PATH_BUF]; + int written = snprintf(init, sizeof(init), "%.*s/__init__.py", (int)size, root); + return written <= 0 || (size_t)written >= sizeof(init) || python_module_file(ctx, init); +} + +/* The roots one directory's manifests state. Found is the module they agree on. */ +static bool python_project_roots(const cbm_pipeline_ctx_t *ctx, CBMHashTable *map, + const char *directory, const char *normalized, int head, + const cbm_gbuf_node_t **found, bool *hidden) { + const char *manifest; + for (size_t m = 0; (manifest = cbm_py_layout_manifest_at(m)) != NULL; m++) { + const char *roots = node_fact(map, "py-project", directory, manifest); + if (!roots || !strcmp(roots, ambiguous_pkgmap_value)) + continue; + for (const char *root = roots; *root;) { + size_t size = strcspn(root, "\n"); + char path[PKGMAP_PATH_BUF]; + int written = snprintf(path, sizeof(path), "%.*s", (int)size, root); + root += size + (root[size] != '\0'); + if (written <= 0 || (size_t)written >= sizeof(path) || + python_root_is_package(ctx, path, size)) + continue; + + bool blocked = false; + const cbm_gbuf_node_t *module = python_under(ctx, path, normalized, head, &blocked); + if (module && *found && module != *found) + return false; + *found = module ? module : *found; + *hidden = *hidden || blocked; + } + } + return true; +} + +/* Nothing nearer has the name, so it is what an installed project provides: the + * file's own project first, then each enclosing one, as an example inside a project + * depends on that project. Work is the path's depth times the fixed manifest names. */ +static const cbm_gbuf_node_t *python_installed(const cbm_pipeline_ctx_t *ctx, + const char *source_rel, const char *normalized, + int head, const cbm_gbuf_node_t *rooted) { + CBMHashTable *map = cbm_pipeline_get_pkgmap(); + if (!map || cbm_py_layout_is_stdlib(normalized, (size_t)head)) + return rooted; + + char directory[PKGMAP_PATH_BUF]; + int size = snprintf(directory, sizeof(directory), "%s", source_rel); + if (size <= 0 || (size_t)size >= sizeof(directory)) + return rooted; + node_parent_path(directory); + const cbm_gbuf_node_t *found = NULL; + bool hidden = false; + for (;;) { + /* The repository root is searched first whatever the file: a module there stands + * before the root manifest's own roots, as it does for a type checker. */ + if (!directory[0] && rooted) + return rooted; + if (!python_project_roots(ctx, map, directory, normalized, head, &found, &hidden)) + return NULL; + if (found || hidden || !directory[0]) + break; + node_parent_path(directory); + } + if (!found && !hidden && !rooted) { + /* A sibling member of a uv workspace enclosing the file. */ + char head_name[PKGMAP_PATH_BUF]; + int written = snprintf(head_name, sizeof(head_name), "%.*s", head, normalized); + if (written > 0 && (size_t)written < sizeof(head_name)) { + snprintf(directory, sizeof(directory), "%s", source_rel); + for (node_parent_path(directory);; node_parent_path(directory)) { + const char *root = node_fact(map, "py-workspace", directory, NULL) + ? node_fact(map, "py-member-head", directory, head_name) + : NULL; + if (root && strcmp(root, ambiguous_pkgmap_value)) { + found = python_under(ctx, root, normalized, head, &hidden); + break; + } + if (!directory[0]) + break; + } + } + } + if (!found) + return hidden ? NULL : rooted; + return !rooted || rooted == found ? found : NULL; +} + /* An absolute import, given what it names relative to the repo root. Python takes the first * match on sys.path, and only the file's own search directory is known. Two different * modules leave the choice to a path we cannot see, except for the file's own package, @@ -3755,24 +4064,30 @@ static const cbm_gbuf_node_t *python_on_search_path(const cbm_pipeline_ctx_t *ct return rooted; if (!python_search_path(ctx, source_rel, &search)) return NULL; - if (!search.base[0]) - return rooted; - char near_path[PKGMAP_PATH_BUF], head_path[PKGMAP_PATH_BUF]; int head = (int)strcspn(normalized, "/"); - int near_size = snprintf(near_path, sizeof(near_path), "%s/%s", search.base, normalized); - int head_size = - snprintf(head_path, sizeof(head_path), "%s/%.*s", search.base, head, normalized); - if (near_size <= 0 || (size_t)near_size >= sizeof(near_path) || head_size <= 0 || - (size_t)head_size >= sizeof(head_path)) - return NULL; - const cbm_gbuf_node_t *found = python_module_at_path(ctx, near_path); - if (found) { - bool own = - (int)strlen(search.own) == head && !strncmp(search.own, normalized, (size_t)head); - return !rooted || own ? found : NULL; + bool hidden = false; + if (search.root[0]) { + const cbm_gbuf_node_t *found = python_under(ctx, search.root, normalized, head, &hidden); + bool own = (int)strlen(search.root_own) == head && + !strncmp(search.root_own, normalized, (size_t)head); + if (found) + return !rooted || own ? found : NULL; + if (hidden || own) + return NULL; } - /* A module of that first name here hides the one at the root, whatever it lacks. */ - return python_module_at_path(ctx, head_path) ? NULL : rooted; + /* Inside a package below a source root, the package's parent is not on sys.path. */ + if (search.base[0] && (!search.root[0] || !search.own[0])) { + const cbm_gbuf_node_t *found = python_under(ctx, search.base, normalized, head, &hidden); + if (found) { + bool own = + (int)strlen(search.own) == head && !strncmp(search.own, normalized, (size_t)head); + return !rooted || own ? found : NULL; + } + /* A module of that first name here hides the one at the root, whatever it lacks. */ + if (hidden) + return NULL; + } + return python_installed(ctx, source_rel, normalized, head, rooted); } static const cbm_gbuf_node_t *python_exact_module(const cbm_pipeline_ctx_t *ctx, diff --git a/src/pipeline/pass_resolver.c b/src/pipeline/pass_resolver.c index 07286da36..4610f93c8 100644 --- a/src/pipeline/pass_resolver.c +++ b/src/pipeline/pass_resolver.c @@ -13,6 +13,7 @@ #include "resolve/lookup.h" #include "resolve/policy.h" #include "resolve/py_members.h" + #include "resolve/rust_names.h" #include "resolve/snapshot.h" #include "resolve/solver.h" @@ -117,6 +118,8 @@ typedef struct { const char *module_qn; cbm_resolve_proof_source_t source; bool complete, ambiguous, python_mutated; + bool python_audited; /* every statement was seen, whatever __getattr__ answers besides */ + bool python_star_chosen; /* a condition chooses among its stars: the binder stopped at one */ const char **python_names; /* module-level binders, sorted, parallel to python_name_at */ uint32_t *python_name_at; size_t python_name_count; @@ -146,11 +149,14 @@ typedef struct { } python_nested_list_t; typedef struct python_pending_import { - const char *key, *slot, *parent_file, *child_file, *member, *escape_root, *escape_slot; + const char *key, *slot, *parent_file, *child_file, *member, *write_root, *write_slot; const cbm_resolve_proof_t *audit; const char *loaded_module; const cbm_resolve_proof_t *load_audit; - bool module, audited, escaped, escape_module; + const char *source_file; + const char **writes; /* dotted paths stored through this import, NULL for any name */ + size_t write_count; + bool module, audited, write_module, in_parent; struct python_pending_import *next; } python_pending_import_t; @@ -190,6 +196,8 @@ struct cbm_shared_resolver { bool python_unknown_namespace_escape; cbm_ecma_table_t *ecma; CBMHashTable *python_star_provided; /* "." a closed star import binds */ + CBMHashTable *python_written; /* "." a statement stores or deletes */ + CBMHashTable *python_parent_loaded; /* a submodule its package's own __init__ loads */ CBMHashTable *python_nested; /* file -> python_nested_list_t */ cbm_py_members_t *python_members; CBMHashTable *http_defaults, *http_sites, *http_unsafe; @@ -387,10 +395,52 @@ static bool declarations_coexist(CBMLanguage language) { } } +/* A typing.overload stub states a signature for the def that follows; with it, one declaration. + * The binding audit proves the decorator is the real one before anything resolves to the name. */ +static bool python_overload_stub_definition(const CBMDefinition *definition) { + for (const char *const *decorator = definition->decorators; decorator && *decorator; + decorator++) { + const char *dot = strrchr(*decorator, '.'); + const char *name = dot ? dot + 1 : *decorator; + if (*name == '@') + name++; + if (!strcmp(name, "overload")) + return true; + } + return false; +} + +/* `@x.setter def x`: the same property as the def it follows, and its one graph node keeps + * this span. */ +static bool python_accessor_definition(const CBMDefinition *definition) { + size_t length = definition->name ? strlen(definition->name) : 0; + for (const char *const *decorator = definition->decorators; length && decorator && *decorator; + decorator++) { + const char *text = **decorator == '@' ? *decorator + 1 : *decorator; + if (strncmp(text, definition->name, length) || text[length] != '.') + continue; + const char *member = text + length + 1; + if (!strcmp(member, "setter") || !strcmp(member, "getter") || !strcmp(member, "deleter")) + return true; + } + return false; +} + static bool add_target(cbm_shared_resolver_t *resolver, const CBMDefinition *definition, const char *file, CBMLanguage language) { if (!definition->qualified_name || !definition->label || !file) return true; + if (language == CBM_LANG_PYTHON && python_overload_stub_definition(definition)) + return true; + if (language == CBM_LANG_PYTHON && python_accessor_definition(definition)) { + target_proof_t *property = cbm_ht_get(resolver->targets, definition->qualified_name); + if (property && !strcmp(property->file, file) && + !strcmp(property->label, definition->label)) { + property->start_line = definition->start_line; + property->end_line = definition->end_line; + return true; + } + } target_proof_t *prior = cbm_ht_get(resolver->targets, definition->qualified_name); if (prior && prior->rust && language == CBM_LANG_RUST) { prior->ambiguous |= strcmp(prior->file, file) != 0; @@ -471,11 +521,16 @@ static bool owner_kind(TSNode node, const char *kind) { } static bool note_owner_branch(cbm_shared_resolver_t *resolver, const CBMFileResult *result, - const char *file, const char *source, size_t length, TSNode group, - TSNode block, unsigned branch) { - uint32_t count = ts_node_named_child_count(block); - for (uint32_t i = 0; i < count; i++) { - TSNode function = ts_node_named_child(block, i); + const char *file, const char *source, size_t length, + const char *scope, TSNode group, TSNode block, unsigned branch) { + TSTreeCursor cursor = ts_tree_cursor_new(block); + bool ok = true; + for (bool more = ts_tree_cursor_goto_first_child(&cursor); ok && more; + more = ts_tree_cursor_goto_next_sibling(&cursor)) { + TSNode statement = ts_tree_cursor_current_node(&cursor); + TSNode function = owner_kind(statement, "decorated_definition") + ? owner_field(statement, "definition") + : statement; TSNode name = owner_field(function, "name"), body = owner_field(function, "body"); if (!owner_kind(function, "function_definition") || !owner_kind(name, "identifier") || !owner_kind(body, "block")) @@ -483,54 +538,115 @@ static bool note_owner_branch(cbm_shared_resolver_t *resolver, const CBMFileResu uint32_t start = ts_node_start_byte(name), end = ts_node_end_byte(name); if (end <= start || end > length || end - start > INT_MAX) continue; - const char *qn = cbm_arena_sprintf(&resolver->arena, "%s.%.*s", result->module_qn, - (int)(end - start), source + start); - if (!qn) - return false; + const char *qn = cbm_arena_sprintf(&resolver->arena, "%s.%.*s", scope, (int)(end - start), + source + start); + if (!qn) { + ok = false; + break; + } target_proof_t *target = cbm_ht_get(resolver->targets, qn); if (!target || !target->ambiguous || target->declarations != 2 || - strcmp(target->file, file) || strcmp(target->label, "Function") || + strcmp(target->file, file) || + (strcmp(target->label, "Function") && strcmp(target->label, "Method")) || target->start_line == target->second_start_line) continue; uint32_t first = ts_node_start_point(function).row + 1; + uint32_t outer = ts_node_start_point(statement).row + 1; uint32_t last = ts_node_end_point(function).row + 1; unsigned member; - if (first == target->start_line && last == target->end_line) + if ((first == target->start_line || outer == target->start_line) && + last == target->end_line) member = 0; - else if (first == target->second_start_line && last == target->second_end_line) + else if ((first == target->second_start_line || outer == target->second_start_line) && + last == target->second_end_line) member = 1; else continue; if (!target->conditional) { conditional_owner_t *owner = cbm_arena_calloc(&resolver->arena, sizeof(*owner)); - if (!owner) - return false; + if (!owner) { + ok = false; + break; + } owner->content = cbm_arena_strdup(&resolver->arena, result->source_content_id[CBM_SOURCE_ORIGIN_RAW]); owner->configuration = cbm_arena_strdup(&resolver->arena, result->source_configuration_id); - if (!owner->content || !owner->configuration) - return false; + if (!owner->content || !owner->configuration) { + ok = false; + break; + } target->conditional = owner; } conditional_owner_t *owner = target->conditional; owner->invalid |= owner->members[member].branch != 0; - owner->members[member] = (owner_variant_t){.start_line = first, - .end_line = last, - .start_byte = ts_node_start_byte(function), - .end_byte = ts_node_end_byte(function), - .body_start = ts_node_start_byte(body), - .body_end = ts_node_end_byte(body), - .group_start = ts_node_start_byte(group), - .group_end = ts_node_end_byte(group), - .branch = branch}; + owner->members[member] = (owner_variant_t){ + .start_line = + first == target->start_line || first == target->second_start_line ? first : outer, + .end_line = last, + .start_byte = ts_node_start_byte(function), + .end_byte = ts_node_end_byte(function), + .body_start = ts_node_start_byte(body), + .body_end = ts_node_end_byte(body), + .group_start = ts_node_start_byte(group), + .group_end = ts_node_end_byte(group), + .branch = branch}; } - return true; + ts_tree_cursor_delete(&cursor); + return ok; +} + +/* An if/else whose two arms each declare a name: one declaration of it, whichever arm runs. */ +static bool note_owner_group(cbm_shared_resolver_t *resolver, const CBMFileResult *result, + const char *file, const char *source, size_t length, const char *scope, + TSNode group) { + TSNode first = owner_field(group, "consequence"); + TSNode alternative = owner_field(group, "alternative"); + TSNode second = owner_field(alternative, "body"); + if (!owner_kind(first, "block") || !owner_kind(alternative, "else_clause") || + !owner_kind(second, "block")) + return true; + return note_owner_branch(resolver, result, file, source, length, scope, group, first, 1) && + note_owner_branch(resolver, result, file, source, length, scope, group, second, 2); +} + +/* Audit one raw syntax tree in linear time: each if/else directly in a scope's body, where the + * scope is the module or a def or class reached through bodies. elif, nested groups, other + * languages and cross-file build variants remain quality gaps. */ +static bool note_conditional_owners_in(cbm_shared_resolver_t *resolver, const CBMFileResult *result, + const char *file, const char *source, size_t length, + const char *scope, TSNode body, unsigned depth) { + /* A cursor, not an index loop: a fetch by index walks from the first child. */ + bool ok = depth < 32; + TSTreeCursor cursor = ts_tree_cursor_new(body); + for (bool more = ok && ts_tree_cursor_goto_first_child(&cursor); ok && more; + more = ts_tree_cursor_goto_next_sibling(&cursor)) { + TSNode statement = ts_tree_cursor_current_node(&cursor); + if (owner_kind(statement, "if_statement")) { + ok = note_owner_group(resolver, result, file, source, length, scope, statement); + continue; + } + TSNode definition = owner_kind(statement, "decorated_definition") + ? owner_field(statement, "definition") + : statement; + if (!owner_kind(definition, "function_definition") && + !owner_kind(definition, "class_definition")) + continue; + TSNode name = owner_field(definition, "name"), inner = owner_field(definition, "body"); + if (!owner_kind(name, "identifier") || !owner_kind(inner, "block")) + continue; + uint32_t start = ts_node_start_byte(name), end = ts_node_end_byte(name); + if (end <= start || end > length) + continue; + const char *child = cbm_arena_sprintf(&resolver->arena, "%s.%.*s", scope, + (int)(end - start), source + start); + ok = child && note_conditional_owners_in(resolver, result, file, source, length, child, + inner, depth + 1); + } + ts_tree_cursor_delete(&cursor); + return ok; } -/* Audit one raw syntax tree in linear time. This deliberately supports only - * direct module-level if/else bodies. elif, nested groups, decorated functions, - * other languages and cross-file build variants remain quality gaps. */ static bool note_conditional_owners(cbm_shared_resolver_t *resolver, const CBMFileResult *result, const char *file, CBMLanguage language, const char *source, size_t length) { @@ -541,26 +657,8 @@ static bool note_conditional_owners(cbm_shared_resolver_t *resolver, const CBMFi TSNode root = ts_tree_root_node(result->cached_tree); if (!owner_kind(root, "module") || ts_node_has_error(root)) return true; - /* A cursor, not an index loop: a fetch by index walks from the first child. */ - bool ok = true; - TSTreeCursor cursor = ts_tree_cursor_new(root); - if (ts_tree_cursor_goto_first_child(&cursor)) { - do { - TSNode group = ts_tree_cursor_current_node(&cursor); - if (!owner_kind(group, "if_statement")) - continue; - TSNode first = owner_field(group, "consequence"); - TSNode alternative = owner_field(group, "alternative"); - TSNode second = owner_field(alternative, "body"); - if (!owner_kind(first, "block") || !owner_kind(alternative, "else_clause") || - !owner_kind(second, "block")) - continue; - ok = note_owner_branch(resolver, result, file, source, length, group, first, 1) && - note_owner_branch(resolver, result, file, source, length, group, second, 2); - } while (ok && ts_tree_cursor_goto_next_sibling(&cursor)); - } - ts_tree_cursor_delete(&cursor); - return ok; + return note_conditional_owners_in(resolver, result, file, source, length, result->module_qn, + root, 0); } static bool target_materialized(const file_resolver_t *resolver, const cbm_gbuf_node_t *node) { @@ -951,8 +1049,13 @@ static bool capture_python_stars(cbm_shared_resolver_t *resolver, const cbm_pipe entry->python_stars = cbm_arena_calloc(&resolver->arena, count * sizeof(*entry->python_stars)); if (!entry->python_stars) return false; + size_t kept = 0; for (size_t i = 0; i < count; i++) { const cbm_resolve_python_star_t *star = cbm_resolve_binding_python_star_at(index, i); + /* Under TYPE_CHECKING it binds nothing at run time, and only some modules have another + * way to have those names. */ + if (star->annotation_only && !cbm_resolve_binding_python_names_come_at_run_time(index)) + continue; CBMImport path = {.local_name = star->module_path, .module_path = star->module_path, .kind = CBM_IMPORT_KIND_PYTHON_MODULE_ALIAS}; @@ -964,9 +1067,9 @@ static bool capture_python_stars(cbm_shared_resolver_t *resolver, const cbm_pipe cbm_free(CBM_MEM_CLASS_SEMANTIC, slot); if (lost) return false; - entry->python_stars[i] = (python_star_source_t){.module = copy, .at = star->at}; + entry->python_stars[kept++] = (python_star_source_t){.module = copy, .at = star->at}; } - entry->python_star_count = count; + entry->python_star_count = kept; return true; } @@ -995,6 +1098,12 @@ static bool capture_python_namespace(cbm_shared_resolver_t *resolver, const cbm_ entry->source = context.source; entry->module_qn = module; entry->complete = cbm_resolve_binding_python_namespace_closed(index); + entry->python_audited = + cbm_resolve_binding_status(index) && + !strcmp(cbm_resolve_binding_status(index), "binding_complete_supported_subset"); + entry->python_star_chosen = + cbm_resolve_binding_status(index) && + !strcmp(cbm_resolve_binding_status(index), "conditional_wildcard_import"); size_t count = cbm_resolve_binding_python_namespace_size(index); if (count > SIZE_MAX / sizeof(*entry->python_names)) return false; @@ -1070,9 +1179,55 @@ static bool python_loaded_module(cbm_shared_resolver_t *resolver, const cbm_pipe return *out != NULL; } +/* What this file writes through the module the import bound. */ +static bool python_note_import_writes(cbm_shared_resolver_t *resolver, + const cbm_pipeline_ctx_t *ctx, + const cbm_resolve_binding_index_t *index, const char *file, + const CBMImport *import, python_pending_import_t *entry) { + const char *member = NULL; + while (cbm_resolve_binding_python_import_write(index, import->local_name, entry->write_count, + &member)) + entry->write_count++; + if (!entry->write_count) + return true; + entry->writes = cbm_arena_alloc(&resolver->arena, entry->write_count * sizeof(*entry->writes)); + if (!entry->writes) + return false; + for (size_t i = 0; i < entry->write_count; i++) { + cbm_resolve_binding_python_import_write(index, import->local_name, i, &member); + entry->writes[i] = member ? cbm_arena_strdup(&resolver->arena, member) : NULL; + if (member && !entry->writes[i]) + return false; + } + char *effect = cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, import, + &entry->write_module); + if (effect) { + entry->write_slot = cbm_arena_strdup(&resolver->arena, effect); + cbm_free(CBM_MEM_CLASS_SEMANTIC, effect); + if (!entry->write_slot) + return false; + } + if (import->kind == CBM_IMPORT_KIND_PYTHON_MODULE_ROOT && import->local_name) { + CBMImport root = *import; + root.module_path = import->local_name; + root.kind = CBM_IMPORT_KIND_PYTHON_MODULE_ALIAS; + bool is_module = false; + char *bound = + cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, &root, &is_module); + if (bound) { + entry->write_root = cbm_arena_strdup(&resolver->arena, bound); + cbm_free(CBM_MEM_CLASS_SEMANTIC, bound); + if (!entry->write_root) + return false; + } + } + return true; +} + static bool defer_python_import(cbm_shared_resolver_t *resolver, const cbm_pipeline_ctx_t *ctx, - const char *file, const CBMImport *import, const char *key, - const char *slot, bool module, bool audited, bool escaped, + const cbm_resolve_binding_index_t *index, const char *file, + const CBMImport *import, const char *key, const char *slot, + bool module, bool audited, bool runs, const cbm_resolve_proof_t *audit, const char *loaded_module, const cbm_resolve_proof_t *load_audit) { python_pending_import_t *entry = cbm_arena_calloc(&resolver->arena, sizeof(*entry)); @@ -1082,36 +1237,14 @@ static bool defer_python_import(cbm_shared_resolver_t *resolver, const cbm_pipel entry->slot = slot ? cbm_arena_strdup(&resolver->arena, slot) : NULL; entry->module = module; entry->audited = audited; - entry->escaped = escaped; entry->audit = audit; entry->loaded_module = loaded_module; entry->load_audit = load_audit; - if (!entry->key || (slot && !entry->slot)) + entry->source_file = cbm_arena_strdup(&resolver->arena, file); + if (!entry->key || (slot && !entry->slot) || !entry->source_file) + return false; + if (runs && !python_note_import_writes(resolver, ctx, index, file, import, entry)) return false; - if (escaped) { - char *effect = cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, import, - &entry->escape_module); - if (effect) { - entry->escape_slot = cbm_arena_strdup(&resolver->arena, effect); - cbm_free(CBM_MEM_CLASS_SEMANTIC, effect); - if (!entry->escape_slot) - return false; - } - } - if (escaped && import->kind == CBM_IMPORT_KIND_PYTHON_MODULE_ROOT && import->local_name) { - CBMImport root = *import; - root.module_path = import->local_name; - root.kind = CBM_IMPORT_KIND_PYTHON_MODULE_ALIAS; - bool is_module = false; - char *bound = - cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, &root, &is_module); - if (bound) { - entry->escape_root = cbm_arena_strdup(&resolver->arena, bound); - cbm_free(CBM_MEM_CLASS_SEMANTIC, bound); - if (!entry->escape_root) - return false; - } - } const cbm_gbuf_node_t *parent, *child; if (audited && !module && cbm_pipeline_python_submodule_candidate(ctx->gbuf, ctx->project_name, file, import, &parent, @@ -1119,6 +1252,8 @@ static bool defer_python_import(cbm_shared_resolver_t *resolver, const cbm_pipel entry->parent_file = cbm_arena_strdup(&resolver->arena, parent->file_path); entry->child_file = cbm_arena_strdup(&resolver->arena, child->file_path); entry->member = cbm_arena_strdup(&resolver->arena, strrchr(import->module_path, '.') + 1); + entry->in_parent = !strcmp(parent->file_path, file) && import->local_name && + !strcmp(import->local_name, entry->member); if (!entry->parent_file || !entry->child_file || !entry->member) return false; } @@ -1511,10 +1646,8 @@ static bool python_export_import(cbm_shared_resolver_t *resolver, const cbm_pipe "python_explicit_module_load_barrier", import) : NULL; ok = ok && (!loaded || load_audit) && - defer_python_import( - resolver, ctx, file, import, fact_key, slot, module, audited, - runs && cbm_resolve_binding_python_import_escaped(index, import->local_name), audit, - loaded, load_audit); + defer_python_import(resolver, ctx, index, file, import, fact_key, slot, module, audited, + runs, audit, loaded, load_audit); /* A plain dotted import binds only a package root, not the loaded leaf * under that root's exported name. Exact module-path uses still retain * the private import fact above. */ @@ -1544,14 +1677,14 @@ static const char *python_member_import_key(void *context, CBMArena *arena, const cbm_resolve_python_nested_import_t *nested, const char *chain, const char **rest) { python_member_imports_t *lookup = context; - size_t dotted = strcspn(chain, "("); + size_t dotted = strcspn(chain, "([@"); if (nested) { const char *base = nested->import.kind == CBM_IMPORT_KIND_PYTHON_MODULE_ROOT ? nested->import.module_path : nested->import.local_name; size_t length = strlen(base); if (length > dotted || strncmp(chain, base, length) || - (chain[length] && chain[length] != '.' && chain[length] != '(')) + (chain[length] && !strchr(".([@", chain[length]))) return NULL; *rest = chain + length; return python_nested_key(arena, lookup->file, nested->at, nested->import.local_name); @@ -1692,8 +1825,9 @@ static bool export_file(cbm_shared_resolver_t *resolver, const cbm_pipeline_ctx_ const CBMDefinition *definition = &result->defs.items[d]; /* Indexing child.py does not execute it or overwrite an explicit child * value in __init__.py. Actual loads are separate effect observations. */ - if (file->language == CBM_LANG_PYTHON && definition->label && - !strcmp(definition->label, "Module")) + if (file->language == CBM_LANG_PYTHON && + ((definition->label && !strcmp(definition->label, "Module")) || + python_overload_stub_definition(definition) || python_accessor_definition(definition))) continue; const char *qn = definition->qualified_name; if (qn && qn[0]) { @@ -1737,7 +1871,12 @@ static bool export_file(cbm_shared_resolver_t *resolver, const cbm_pipeline_ctx_ python_export_import( resolver, ctx, result, file->rel_path, &arena, index, &item->import, key, cbm_resolve_binding_python_nested_audited(index, item), - item->scope == 0 && !item->annotation_only, !item->annotation_only); + /* A TYPE_CHECKING import runs nothing. In a module whose names can come at + * run time it is what the module states that name to be: celery and every + * PEP 562 lazy package rely on it. Any other module has no such name then. */ + item->scope == 0 && (!item->annotation_only || + cbm_resolve_binding_python_names_come_at_run_time(index)), + !item->annotation_only); } if (ok && file->language == CBM_LANG_PYTHON && cbm_resolve_binding_status(index) && !strcmp(cbm_resolve_binding_status(index), "binding_complete_supported_subset")) { @@ -1773,14 +1912,24 @@ static int compare_export(const void *left, const void *right) { return strcmp((*(export_proof_t *const *)left)->key, (*(export_proof_t *const *)right)->key); } +/* `from pkg import child` finds the submodule when pkg's namespace has no such name. In + * pkg's own __init__ the statement is itself that name's binder, and it runs before the + * name exists, so a binder there of its own does not count. */ static bool python_namespace_absent(cbm_shared_resolver_t *resolver, - const namespace_proof_t *parent, const char *name) { - if (!parent || !parent->complete || parent->ambiguous || parent->python_mutated || !name || + const namespace_proof_t *parent, const char *name, + bool in_parent) { + if (!parent || parent->ambiguous || parent->python_mutated || !name || parent->python_open_barrier) return false; /* A closed star import binds this name as a value, not the submodule. */ const char *qualified = cbm_arena_sprintf(&resolver->arena, "%s.%s", parent->module_qn, name); - if (!qualified || cbm_ht_get(resolver->python_star_provided, qualified)) + if (!qualified || cbm_ht_get(resolver->python_star_provided, qualified) || + cbm_ht_get(resolver->python_written, qualified)) + return false; + /* __getattr__ answers only for a name the module lacks. A child its own __init__ loads is + * set on it by that load, so it never gets to answer for this one. */ + if (!parent->complete && + !(parent->python_audited && cbm_ht_get(resolver->python_parent_loaded, qualified))) return false; size_t low = 0, high = parent->python_name_count; while (low < high) { @@ -1790,7 +1939,11 @@ static bool python_namespace_absent(cbm_shared_resolver_t *resolver, else high = middle; } - return low == parent->python_name_count || strcmp(parent->python_names[low], name) != 0; + size_t binders = 0; + while (low + binders < parent->python_name_count && + !strcmp(parent->python_names[low + binders], name)) + binders++; + return binders == 0 || (in_parent && binders == 1); } static const cbm_resolve_proof_t *python_submodule_proof(cbm_shared_resolver_t *resolver, @@ -1838,13 +1991,36 @@ static void invalidate_python_namespace(cbm_shared_resolver_t *resolver, const c * same-leaf files never invalidate one another. Work follows imported * namespace prefixes, using hash lookups rather than a files-by-import scan. */ static bool collect_python_module_loads(cbm_shared_resolver_t *resolver) { + /* `from . import child` in the parent's own __init__ binds the very module a load installs + * there, so that load competes with nothing. */ + CBMHashTable *own_children = cbm_ht_create(32); + if (!own_children) + return false; + for (const python_pending_import_t *import = resolver->python_imports; import; + import = import->next) { + /* A load by the parent's own __init__: a from-import of the child, or of something in it. + */ + const namespace_proof_t *source = cbm_ht_get(resolver->namespaces, import->source_file); + const char *dot = import->loaded_module ? strrchr(import->loaded_module, '.') : NULL; + if (source && dot && strlen(source->module_qn) == (size_t)(dot - import->loaded_module) && + !strncmp(source->module_qn, import->loaded_module, + (size_t)(dot - import->loaded_module))) + cbm_ht_set(resolver->python_parent_loaded, import->loaded_module, resolver); + const char *key = import->in_parent && import->slot ? import->slot : NULL; + if (key) { + cbm_ht_set(own_children, key, resolver); + cbm_ht_set(resolver->python_parent_loaded, key, resolver); + } + } for (const python_pending_import_t *import = resolver->python_imports; import; import = import->next) { if (!import->loaded_module) continue; char *module = cbm_arena_strdup(&resolver->arena, import->loaded_module); - if (!module || !import->load_audit) + if (!module || !import->load_audit) { + cbm_ht_free(own_children); return false; + } for (;;) { const namespace_proof_t *loaded = cbm_ht_get(resolver->python_namespaces, module); target_proof_t *declaration = cbm_ht_get(resolver->targets, module); @@ -1855,9 +2031,12 @@ static bool collect_python_module_loads(cbm_shared_resolver_t *resolver) { /* Keep actual load support beside any value it can overwrite. A * source namespace alone is never a competing value declaration. */ if (loaded && cbm_ht_get(resolver->exports, module) && + !cbm_ht_get(own_children, module) && !add_export(resolver, module, loaded->source.file, 0, false, NULL, - import->load_audit)) + import->load_audit)) { + cbm_ht_free(own_children); return false; + } /* The load installs this one child on its parent: the same module * identity a submodule import names, so it closes no other name. */ char *dot = strrchr(module, '.'); @@ -1866,21 +2045,35 @@ static bool collect_python_module_loads(cbm_shared_resolver_t *resolver) { *dot = '\0'; } } + cbm_ht_free(own_children); return true; } -static void collect_python_namespace_effects(cbm_shared_resolver_t *resolver) { +/* A statement that stores through an imported module changes the name it spells, and + * no other: `django.__file__ = None` in a test says nothing about django.forms. Only a + * write that can reach any name takes the module's whole absence proof. */ +static bool collect_python_namespace_effects(cbm_shared_resolver_t *resolver) { for (const python_pending_import_t *import = resolver->python_imports; import; import = import->next) { - if (!import->escaped) - continue; - /* A module object handed to unknown code can be written through, so - * its own namespace loses the absence proof. A value escaping does not - * reach any other module's namespace. */ - invalidate_python_namespace(resolver, import->escape_root); - if (import->escape_module) - invalidate_python_namespace(resolver, import->escape_slot); + const char *base = import->write_root ? import->write_root : import->write_slot; + for (size_t i = 0; i < import->write_count; i++) { + if (!import->writes[i]) { + invalidate_python_namespace(resolver, import->write_root); + if (import->write_module) + invalidate_python_namespace(resolver, import->write_slot); + continue; + } + if (!base) + continue; + const char *key = cbm_arena_sprintf(&resolver->arena, "%s.%s", base, import->writes[i]); + if (!key) + return false; + cbm_ht_set(resolver->python_written, key, resolver); + if (cbm_ht_get(resolver->python_written, key) != resolver) + return false; + } } + return true; } /* The names `from module import *` binds. Unknown when __all__ is not a static @@ -1888,15 +2081,16 @@ static void collect_python_namespace_effects(cbm_shared_resolver_t *resolver) { static void python_star_interface(cbm_shared_resolver_t *resolver, namespace_proof_t *module) { if (module->python_visible_state == 2) return; - if (module->python_visible_state == 1 || !module->complete || module->ambiguous || - module->python_all_unknown) { + /* A computed __all__ can only narrow what the module binds, and a file that asks a star for + * a name relies on it being there. So the public names stand in for it. */ + if (module->python_visible_state == 1 || !module->complete || module->ambiguous) { module->python_visible_open = true; if (module->python_visible_state == 0) module->python_visible_state = 2; return; } module->python_visible_state = 1; - if (module->python_all_literal) { + if (module->python_all_literal && !module->python_all_unknown) { module->python_visible = module->python_all; module->python_visible_count = module->python_all_count; module->python_visible_state = 2; @@ -1955,7 +2149,9 @@ static bool python_explicit_after(const namespace_proof_t *module, const char *n } /* Last star wins; a later explicit binder wins over any star. An unknown star - * can rebind whatever precedes it, so that stops the walk. */ + * can rebind whatever precedes it, so that stops the walk, and so does a star under a condition: + * the binder stops at the first of `if c: from .a import * else: from .b import *` and never + * sees the other. */ static bool python_expand_module_stars(cbm_shared_resolver_t *resolver, namespace_proof_t *module) { for (size_t s = module->python_star_count; s-- > 0;) { const python_star_source_t *star = &module->python_stars[s]; @@ -1963,7 +2159,7 @@ static bool python_expand_module_stars(cbm_shared_resolver_t *resolver, namespac star->module ? cbm_ht_get(resolver->python_namespaces, star->module) : NULL; if (source) python_star_interface(resolver, source); - if (!source || source->python_visible_open) { + if (!source || source->python_visible_open || (module->python_star_chosen)) { module->python_open_barrier = star->at; for (size_t n = 0; n < module->python_name_count; n++) { if (module->python_name_at[n] >= star->at) @@ -2178,8 +2374,7 @@ static bool decide_python_decorated(cbm_shared_resolver_t *resolver) { static bool settle_python_imports(cbm_shared_resolver_t *resolver) { if (!collect_python_module_loads(resolver)) return false; - collect_python_namespace_effects(resolver); - if (!expand_python_stars(resolver)) + if (!collect_python_namespace_effects(resolver) || !expand_python_stars(resolver)) return false; for (python_pending_import_t *import = resolver->python_imports; import; import = import->next) { @@ -2187,8 +2382,8 @@ static bool settle_python_imports(cbm_shared_resolver_t *resolver) { import->parent_file ? cbm_ht_get(resolver->namespaces, import->parent_file) : NULL; const namespace_proof_t *child = import->child_file ? cbm_ht_get(resolver->namespaces, import->child_file) : NULL; - if (child && !child->ambiguous && !child->python_mutated && - python_namespace_absent(resolver, parent, import->member)) { + if (child && !child->ambiguous && + python_namespace_absent(resolver, parent, import->member, import->in_parent)) { const cbm_resolve_proof_t *proof = python_submodule_proof(resolver, import, parent, child); if (!proof) @@ -2525,8 +2720,10 @@ static bool python_env_module_member(const void *context, CBMArena *arena, const /* An attribute no module-level binder provides reads the loaded submodule. */ namespace_proof_t *parent = cbm_ht_get(resolver->python_namespaces, module); namespace_proof_t *child = cbm_ht_get(resolver->python_namespaces, member); - if (binding.proof || !child || child->ambiguous || !parent || !parent->complete || - parent->ambiguous || parent->python_open_barrier) + if (binding.proof || !child || child->ambiguous || !parent || parent->ambiguous || + parent->python_open_barrier || + (!parent->complete && + !(parent->python_audited && cbm_ht_get(resolver->python_parent_loaded, member)))) return false; *target = child->module_qn; *is_module = true; @@ -2704,6 +2901,8 @@ cbm_shared_resolver_t *cbm_shared_resolver_build(const cbm_pipeline_ctx_t *ctx, resolver->python_queries = cbm_ht_create(128); resolver->python_namespaces = cbm_ht_create(128); resolver->python_star_provided = cbm_ht_create(128); + resolver->python_written = cbm_ht_create(128); + resolver->python_parent_loaded = cbm_ht_create(128); resolver->python_nested = cbm_ht_create(128); resolver->python_members = cbm_py_members_new(); resolver->http_defaults = cbm_ht_create(128); @@ -2724,7 +2923,8 @@ cbm_shared_resolver_t *cbm_shared_resolver_build(const cbm_pipeline_ctx_t *ctx, resolver->complete = true; if (!resolver->exports || !resolver->targets || !resolver->namespaces || !resolver->python_queries || !resolver->python_namespaces || - !resolver->python_star_provided || !resolver->python_nested || !resolver->python_members || + !resolver->python_star_provided || !resolver->python_written || + !resolver->python_parent_loaded || !resolver->python_nested || !resolver->python_members || !resolver->http_defaults || !resolver->http_sites || !resolver->http_unsafe || !resolver->http_candidates || !resolver->http_veto || !resolver->project || !resolver->solver || !resolver->go_types || !resolver->ecma || !resolver->source_table) { diff --git a/src/pipeline/py_layout.c b/src/pipeline/py_layout.c new file mode 100644 index 000000000..d0d6d71eb --- /dev/null +++ b/src/pipeline/py_layout.c @@ -0,0 +1,621 @@ +#include "pipeline/py_layout.h" +#include +#include + +enum { PY_LAYOUT_SECTION_MAX = 64 }; + +bool cbm_py_layout_is_manifest(const char *basename) { + return basename && (!strcmp(basename, "pyproject.toml") || !strcmp(basename, "setup.cfg") || + !strcmp(basename, "setup.py") || !strcmp(basename, "tox.ini") || + !strcmp(basename, "pytest.ini")); +} + +const char *cbm_py_layout_manifest_at(size_t index) { + static const char *const names[] = {"pyproject.toml", "setup.cfg", "setup.py", "tox.ini", + "pytest.ini"}; + return index < sizeof(names) / sizeof(names[0]) ? names[index] : NULL; +} + +static const char *py_past_spaces(const char *p, const char *end) { + while (p < end && isspace((unsigned char)*p)) + p++; + return p; +} + +static const char *py_line_end(const char *p, const char *end) { + const char *newline = p < end ? memchr(p, '\n', (size_t)(end - p)) : NULL; + return newline ? newline : end; +} + +static const char *py_skip_blank(const char *p, const char *end) { + while (p < end && (*p == ' ' || *p == '\t' || *p == '\r')) + p++; + return p; +} + +/* ── pyproject.toml ──────────────────────────────────────────────────── */ + +typedef enum { + PY_FORM_STRINGS, /* every string in the value is a root */ + PY_FORM_PARENTS, /* every string names a package: its directory is the root */ + PY_FORM_AFTER_FROM, /* poetry's { include = "x", from = "lib" } */ + PY_FORM_AFTER_EMPTY /* setuptools' { "" = "src" } */ +} py_form_t; + +typedef struct { + const char *section, *key; + py_form_t form; +} py_toml_key_t; + +static const py_toml_key_t py_toml_keys[] = { + {"tool.setuptools.packages.find", "where", PY_FORM_STRINGS}, + {"tool.setuptools", "package-dir", PY_FORM_AFTER_EMPTY}, + {"tool.setuptools.package-dir", "\"\"", PY_FORM_STRINGS}, + {"tool.poetry", "packages", PY_FORM_AFTER_FROM}, + {"tool.poetry.packages", "from", PY_FORM_STRINGS}, + {"tool.hatch.build.targets.wheel", "packages", PY_FORM_PARENTS}, + {"tool.hatch.build", "packages", PY_FORM_PARENTS}, + {"tool.hatch.build.targets.wheel", "sources", PY_FORM_STRINGS}, + {"tool.hatch.build", "sources", PY_FORM_STRINGS}, + {"tool.pdm.build", "package-dir", PY_FORM_STRINGS}, + {"tool.maturin", "python-source", PY_FORM_STRINGS}, + {"tool.uv.build-backend", "module-root", PY_FORM_STRINGS}, + {"tool.pytest.ini_options", "pythonpath", PY_FORM_STRINGS}, + {"tool.ruff", "src", PY_FORM_STRINGS}, + {"tool.pyright", "extraPaths", PY_FORM_STRINGS}, + {"tool.mypy", "mypy_path", PY_FORM_STRINGS}, +}; + +/* The end of a value: its line, or past the bracket that closes it. Strings and + * comments cannot close one. */ +static const char *py_toml_value_end(const char *value, const char *end) { + if (value >= end || (*value != '[' && *value != '{')) + return py_line_end(value, end); + int depth = 0; + char quote = 0; + for (const char *p = value; p < end; p++) { + if (quote) { + quote = *p == quote ? 0 : (*p == '\n' ? 0 : quote); + continue; + } + if (*p == '"' || *p == '\'') { + quote = *p; + } else if (*p == '#') { + p = py_line_end(p, end); + if (p >= end) + break; + } else if (*p == '[' || *p == '{') { + depth++; + } else if ((*p == ']' || *p == '}') && --depth == 0) { + return p + 1; + } + } + return end; +} + +/* The strings of a value, in order, comments skipped. */ +static void py_toml_strings(const char *value, const char *end, py_form_t form, + cbm_py_layout_emit_fn emit, void *context) { + bool after_key = false; + for (const char *p = value; p < end; p++) { + if (*p == '#') { + p = py_line_end(p, end); + if (p >= end) + return; + continue; + } + if (form == PY_FORM_AFTER_FROM && end - p >= 4 && !memcmp(p, "from", 4)) { + after_key = true; + p += 3; + continue; + } + if (*p != '"' && *p != '\'') + continue; + const char *close = p + 1; + while (close < end && *close != *p && *close != '\n') + close++; + if (close >= end || *close != *p) + return; + size_t length = (size_t)(close - p - 1); + const char *text = p + 1; + p = close; + switch (form) { + case PY_FORM_STRINGS: + emit(context, text, length); + break; + case PY_FORM_PARENTS: { + size_t cut = length; + while (cut && text[cut - 1] != '/') + cut--; + if (cut > 1) + emit(context, text, cut - 1); + break; + } + case PY_FORM_AFTER_FROM: + if (after_key) + emit(context, text, length); + after_key = false; + break; + case PY_FORM_AFTER_EMPTY: + if (after_key) + emit(context, text, length); + after_key = !length; + break; + } + } +} + +static bool py_toml_section(const char *line, const char *line_end, char *section, size_t size) { + if (line >= line_end || *line != '[') + return false; + const char *p = line; + while (p < line_end && *p == '[') + p++; + const char *close = p; + while (close < line_end && *close != ']') + close++; + size_t length = (size_t)(close - p); + if (length >= size) { + section[0] = '\0'; + return true; + } + memcpy(section, p, length); + section[length] = '\0'; + return true; +} + +static void py_parse_toml(const char *source, const char *end, cbm_py_layout_emit_fn emit, + void *context) { + char section[PY_LAYOUT_SECTION_MAX] = ""; + for (const char *line = source; line < end;) { + const char *start = py_skip_blank(line, end); + const char *line_end = py_line_end(start, end); + const char *next = line_end < end ? line_end + 1 : end; + if (py_toml_section(start, line_end, section, sizeof(section))) { + line = next; + continue; + } + for (size_t i = 0; i < sizeof(py_toml_keys) / sizeof(py_toml_keys[0]); i++) { + const py_toml_key_t *key = &py_toml_keys[i]; + size_t length = strlen(key->key); + if (strcmp(section, key->section) || (size_t)(line_end - start) < length || + memcmp(start, key->key, length)) + continue; + const char *value = py_skip_blank(start + length, line_end); + if (value >= line_end || *value != '=') + continue; + value = py_skip_blank(value + 1, line_end); + const char *value_end = py_toml_value_end(value, end); + if (value < line_end && (*value != '{' || key->form == PY_FORM_AFTER_EMPTY)) + py_toml_strings(value, value_end, key->form, emit, context); + if (value_end > next) + next = py_line_end(value_end, end) < end ? py_line_end(value_end, end) + 1 : end; + break; + } + line = next; + } +} + +/* ── setup.cfg ───────────────────────────────────────────────────────── */ + +static void py_cfg_tokens(const char *p, const char *end, cbm_py_layout_emit_fn emit, + void *context) { + while (p < end) { + while (p < end && isspace((unsigned char)*p)) + p++; + const char *token = p; + while (p < end && !isspace((unsigned char)*p)) + p++; + if (p > token) + emit(context, token, (size_t)(p - token)); + } +} + +/* One line of a value: on the key's line, or a continuation below it. */ +static void py_cfg_value(const char *section, const char *key, size_t key_length, const char *text, + const char *end, cbm_py_layout_emit_fn emit, void *context) { + text = py_skip_blank(text, end); + if (!strcmp(section, "options") && key_length == 11 && !memcmp(key, "package_dir", 11)) { + /* `= src` maps the root package; `name = dir` maps one package only. */ + if (text < end && *text == '=') + py_cfg_tokens(text + 1, end, emit, context); + return; + } + if ((!strcmp(section, "options.packages.find") && key_length == 5 && + !memcmp(key, "where", 5)) || + ((!strcmp(section, "tool:pytest") || !strcmp(section, "pytest")) && key_length == 10 && + !memcmp(key, "pythonpath", 10))) + py_cfg_tokens(text, end, emit, context); +} + +static void py_parse_cfg(const char *source, const char *end, cbm_py_layout_emit_fn emit, + void *context) { + char section[PY_LAYOUT_SECTION_MAX] = ""; + const char *key = NULL; + size_t key_length = 0; + for (const char *line = source; line < end;) { + const char *line_end = py_line_end(line, end); + const char *next = line_end < end ? line_end + 1 : end; + const char *start = py_skip_blank(line, line_end); + if (start >= line_end || *start == '#' || *start == ';') { + line = next; + continue; + } + if (start == line && py_toml_section(start, line_end, section, sizeof(section))) { + key = NULL; + } else if (start != line) { + if (key) + py_cfg_value(section, key, key_length, start, line_end, emit, context); + } else { + const char *equals = memchr(start, '=', (size_t)(line_end - start)); + key = start; + const char *stop = equals ? equals : line_end; + while (stop > key && (stop[-1] == ' ' || stop[-1] == '\t')) + stop--; + key_length = (size_t)(stop - key); + if (equals) + py_cfg_value(section, key, key_length, equals + 1, line_end, emit, context); + } + line = next; + } +} + +/* ── setup.py ────────────────────────────────────────────────────────── */ + +static const char *py_find(const char *p, const char *end, const char *text) { + size_t length = strlen(text); + for (; p && (size_t)(end - p) >= length; p++) { + p = memchr(p, text[0], (size_t)(end - p)); + if (!p || (size_t)(end - p) < length) + return NULL; + if (!memcmp(p, text, length)) + return p; + } + return NULL; +} + +static const char *py_literal(const char *p, const char *end, const char **text, size_t *length) { + p = p < end ? p : end; + while (p < end && isspace((unsigned char)*p)) + p++; + if (p >= end || (*p != '"' && *p != '\'')) + return NULL; + const char *close = p + 1; + while (close < end && *close != *p && *close != '\n') + close++; + if (close >= end || *close != *p) + return NULL; + *text = p + 1; + *length = (size_t)(close - p - 1); + return close + 1; +} + +/* `package_dir={"": "src"}` and `find_packages("src")`, as literals. */ +static void py_parse_setup_py(const char *source, const char *end, cbm_py_layout_emit_fn emit, + void *context) { + const char *text = NULL; + size_t length = 0; + for (const char *p = py_find(source, end, "package_dir"); p; + p = py_find(p + 1, end, "package_dir")) { + const char *q = p + strlen("package_dir"); + while (q < end && isspace((unsigned char)*q)) + q++; + if (q >= end || *q != '=') + continue; + q++; + while (q < end && isspace((unsigned char)*q)) + q++; + if (q >= end || *q != '{') + continue; + q = py_literal(q + 1, end, &text, &length); + if (!q || length) + continue; + while (q < end && isspace((unsigned char)*q)) + q++; + if (q >= end || *q != ':') + continue; + if (py_literal(q + 1, end, &text, &length)) + emit(context, text, length); + } + static const char *const finders[] = {"find_packages(", "find_namespace_packages("}; + for (size_t f = 0; f < sizeof(finders) / sizeof(finders[0]); f++) { + for (const char *p = py_find(source, end, finders[f]); p; + p = py_find(p + 1, end, finders[f])) { + const char *q = py_past_spaces(p + strlen(finders[f]), end); + if (end - q > 5 && !memcmp(q, "where", 5)) { + q = py_past_spaces(q + 5, end); + if (q >= end || *q != '=') + continue; + q++; + } + if (py_literal(q, end, &text, &length)) + emit(context, text, length); + } + } +} + +void cbm_py_layout_parse(const char *basename, const char *source, size_t source_len, + cbm_py_layout_emit_fn emit, void *context) { + const char *end = source + source_len; + if (!strcmp(basename, "pyproject.toml")) + py_parse_toml(source, end, emit, context); + else if (!strcmp(basename, "setup.cfg") || !strcmp(basename, "tox.ini") || + !strcmp(basename, "pytest.ini")) + py_parse_cfg(source, end, emit, context); + else if (!strcmp(basename, "setup.py")) + py_parse_setup_py(source, end, emit, context); +} + +void cbm_py_layout_workspace_members(const char *source, size_t source_len, + cbm_py_layout_emit_fn emit, void *context) { + const char *end = source + source_len; + char section[PY_LAYOUT_SECTION_MAX] = ""; + for (const char *line = source; line < end;) { + const char *start = py_skip_blank(line, end); + const char *line_end = py_line_end(start, end); + const char *next = line_end < end ? line_end + 1 : end; + if (py_toml_section(start, line_end, section, sizeof(section))) { + line = next; + continue; + } + if (!strcmp(section, "tool.uv.workspace") && line_end - start > 7 && + !memcmp(start, "members", 7)) { + const char *value = py_skip_blank(start + 7, line_end); + if (value < line_end && *value == '=') { + value = py_skip_blank(value + 1, line_end); + const char *value_end = py_toml_value_end(value, end); + py_toml_strings(value, value_end, PY_FORM_STRINGS, emit, context); + return; + } + } + line = next; + } +} + +bool cbm_py_layout_member_matches(const char *pattern, const char *directory) { + size_t length = strlen(pattern); + if (length < 2 || strcmp(pattern + length - 2, "/*")) + return !strcmp(pattern, directory); + return !strncmp(pattern, directory, length - 1) && directory[length - 1] && + !strchr(directory + length - 1, '/'); +} + +/* ── Standard library ────────────────────────────────────────────────── */ + +/* Sorted. CPython's sys.stdlib_module_names, top-level public modules. */ +static const char *const py_stdlib_modules[] = { + "abc", + "aifc", + "antigravity", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "genericpath", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "nt", + "ntpath", + "nturl2path", + "numbers", + "opcode", + "operator", + "optparse", + "os", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "pydoc_data", + "pyexpat", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "textwrap", + "this", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "tomllib", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +}; + +bool cbm_py_layout_is_stdlib(const char *name, size_t length) { + size_t low = 0, high = sizeof(py_stdlib_modules) / sizeof(py_stdlib_modules[0]); + while (low < high) { + size_t middle = low + (high - low) / 2; + int order = strncmp(py_stdlib_modules[middle], name, length); + if (!order && py_stdlib_modules[middle][length]) + order = 1; + if (!order) + return true; + if (order < 0) + low = middle + 1; + else + high = middle; + } + return false; +} diff --git a/src/pipeline/py_layout.h b/src/pipeline/py_layout.h new file mode 100644 index 000000000..d403ee163 --- /dev/null +++ b/src/pipeline/py_layout.h @@ -0,0 +1,36 @@ +#ifndef CBM_PIPELINE_PY_LAYOUT_H +#define CBM_PIPELINE_PY_LAYOUT_H + +#include +#include + +/* Where a Python project keeps its importable code, as its manifests state it. + * A file under a stated root is the installed module of that path: src/flask/app.py + * is flask.app, and `import flask` anywhere in the project names it. Manifests are read + * as text and nothing in them is run. */ + +/* pyproject.toml, setup.cfg, setup.py, tox.ini or pytest.ini. */ +bool cbm_py_layout_is_manifest(const char *basename); + +/* The manifests that can sit in one directory, for a lookup per directory. */ +const char *cbm_py_layout_manifest_at(size_t index); + +/* Each root the manifest states, as written, relative to its directory. */ +typedef void (*cbm_py_layout_emit_fn)(void *context, const char *root, size_t length); +void cbm_py_layout_parse(const char *basename, const char *source, size_t source_len, + cbm_py_layout_emit_fn emit, void *context); + +/* Each member pattern of a uv workspace this pyproject.toml declares, as written: its members + * are installed together, so each one's modules are importable from the others. */ +void cbm_py_layout_workspace_members(const char *source, size_t source_len, + cbm_py_layout_emit_fn emit, void *context); + +/* Whether a project directory matches a member pattern, both relative to the workspace: + * an exact path, or a final `*` standing for one directory name. */ +bool cbm_py_layout_member_matches(const char *pattern, const char *directory); + +/* A top-level standard library module. The standard library precedes an installed + * distribution on sys.path, so such a name never reaches a source root. */ +bool cbm_py_layout_is_stdlib(const char *name, size_t length); + +#endif diff --git a/tests/test_main.c b/tests/test_main.c index 92b528ff3..3ba8ff362 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1803,7 +1803,9 @@ extern void suite_resolver_rust(void); extern void suite_resolver_py_types(void); extern void suite_resolver_python(void); extern void suite_resolver_python_members(void); +extern void suite_resolver_python_layout(void); extern void suite_resolver_python_flow(void); +extern void suite_resolver_python_shapes(void); extern void suite_resolver_java(void); extern void suite_resolver_oo(void); extern void suite_resolver_diagnostics(void); @@ -2446,7 +2448,9 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(resolver_py_types); RUN_SELECTED_SUITE(resolver_python); RUN_SELECTED_SUITE(resolver_python_members); + RUN_SELECTED_SUITE(resolver_python_layout); RUN_SELECTED_SUITE(resolver_python_flow); + RUN_SELECTED_SUITE(resolver_python_shapes); RUN_SELECTED_SUITE(resolver_java); RUN_SELECTED_SUITE(resolver_oo); RUN_SELECTED_SUITE(resolver_diagnostics); diff --git a/tests/test_resolver_binding_generic.c b/tests/test_resolver_binding_generic.c index 5d3e2c898..ab06e4730 100644 --- a/tests/test_resolver_binding_generic.c +++ b/tests/test_resolver_binding_generic.c @@ -1119,13 +1119,13 @@ TEST(resolver_binding_python_formatting_keeps_module_escape_and_rebinding_guards cbm_resolve_result_t proof = cbm_resolve_binding_local( index, "target", start, start + strlen("target()"), CBM_SOURCE_ORIGIN_RAW); bool admitted = cbm_resolve_admit(&proof); - bool escaped = cbm_resolve_binding_python_import_escaped(index, "library"); + const char *member = NULL; + bool written = cbm_resolve_binding_python_import_write(index, "library", 0, &member); cbm_arena_destroy(&arena); cbm_free_result(file); - /* The escape is recorded against library; target itself stays proven. */ + /* Formatting a module reads it, and target itself stays proven. */ ASSERT_EQ(admitted, i == 0); - if (i == 0) - ASSERT(escaped); + ASSERT(!written); } PASS(); } @@ -1999,7 +1999,8 @@ static bool python_physical_package_golden(const char *manifest, bool source_lay const char *library_path = source_layout ? "src/sample/utils.py" : "sample/utils.py"; const char *decoy_path = source_layout ? "decoy/utils.py" : "src/sample/utils.py"; const char *initializer = source_layout ? "src/sample/__init__.py" : "sample/__init__.py"; - bool supported = !source_layout || explicit_src; + /* A manifest beside src/ states the layout; its [project].name says nothing. */ + bool supported = !source_layout || explicit_src || manifest; char main[256]; snprintf(main, sizeof(main), "from %ssample.utils import target\ndef caller():\n return target()\n" @@ -2029,7 +2030,7 @@ static bool python_physical_package_golden(const char *manifest, bool source_lay .declaration_text = "def target(value: Optional[str] = None) -> Tuple[str, str]:"}}}; const RQ_Forbidden forbidden = { - .case_id = "distribution-name-does-not-authorize-src-layout", + .case_id = "no-manifest-states-a-src-layout", .site = {.file = "main.py", .caller = "caller", .needle = "target()"}, .relation = "CALLS", .any_target = true}; @@ -2098,6 +2099,7 @@ TEST(resolver_binding_python_distribution_name_is_not_import_root) { static const char other_distribution[] = "[project]\nname = 'unrelated-dist'\n"; for (int driver = 0; driver < 2; driver++) { ASSERT(python_physical_package_golden(NULL, false, false, driver)); + ASSERT(python_physical_package_golden(NULL, true, false, driver)); ASSERT(python_physical_package_golden(manifest, false, false, driver)); ASSERT(python_physical_package_golden(other_distribution, false, false, driver)); ASSERT(python_physical_package_golden(manifest, true, false, driver)); diff --git a/tests/test_resolver_python.c b/tests/test_resolver_python.c index 5522c80b1..92101e8d7 100644 --- a/tests/test_resolver_python.c +++ b/tests/test_resolver_python.c @@ -560,18 +560,6 @@ TEST(resolver_python_submodule_import_survives_a_working_package) { {"main.py", "from pkg import mod\ndef run():\n return mod.real()\n"}}; ASSERT(python_expect("star value", files, 4, "CALLS", "pkg/mod.py", "real", "Function", 0)); } - /* A module object handed to unknown code can be written through, and so can - * a member path that stores. */ - static const char *patchers[] = {"import pkg\ndef patch():\n install(pkg)\n", - "import pkg\npkg.mod = None\n"}; - for (size_t i = 0; i < sizeof(patchers) / sizeof(patchers[0]); i++) { - const python_proof_file_t files[] = { - {"pkg/__init__.py", ""}, - child, - {"patcher.py", patchers[i]}, - {"main.py", "from pkg import mod\ndef run():\n return mod.real()\n"}}; - ASSERT(python_expect(patchers[i], files, 4, "CALLS", "pkg/mod.py", "real", "Function", 0)); - } PASS(); } @@ -1078,6 +1066,9 @@ TEST(resolver_python_receiver_types_need_a_fact) { * rebinds it, and every `self` that is not the method's own, abstains. */ /* A class body's assignment says what a read yields only while nothing can replace it. In * django `self.title` on a model is a str, never the CharField the body assigned. */ +/* A class body's value is the attribute's declared value. django's objects = UserManager() reads + * as what it says under ModelBase; only the standard library's Enum and NamedTuple state that a + * member reads as something else. */ TEST(resolver_python_class_body_values_can_be_replaced) { static const struct { const char *what; @@ -1093,21 +1084,21 @@ TEST(resolver_python_class_body_values_can_be_replaced) { {"own metaclass", "class A(metaclass=Meta):\n title = Field()\n def run(self):\n" " return self.title.clean()\n", - 0}, + 1}, {"metaclass on an ancestor", "class M(metaclass=Meta):\n pass\nclass A(M):\n title = Field()\n" " def run(self):\n return self.title.clean()\n", - 0}, + 1}, {"metaclass on the receiver's own class", "class P:\n title = Field()\nclass A(P, metaclass=Meta):\n def run(self):\n" " return self.title.clean()\n", - 0}, + 1}, {"read from outside, metaclass", - "class A(metaclass=Meta):\n title = Field()\ndef run():\n return A.title.clean()\n", 0}, + "class A(metaclass=Meta):\n title = Field()\ndef run():\n return A.title.clean()\n", 1}, {"__init_subclass__", "class H:\n def __init_subclass__(cls):\n pass\nclass A(H):\n" " title = Field()\n def run(self):\n return self.title.clean()\n", - 0}, + 1}, {"enum member", "from enum import Enum\nclass A(Enum):\n RED = Field()\n def run(self):\n" " return self.RED.clean()\n", @@ -1240,7 +1231,8 @@ TEST(resolver_python_rebound_receivers_abstain) { {"tuple target", "def run(pair):\n s = Store()\n s, t = pair\n return s.get()\n", 0}, {"star target", "def run(items):\n s = Store()\n *s, t = items\n return s.get()\n", 0}, {"deleted", "def run():\n s = Store()\n del s\n return s.get()\n", 0}, - {"parameter default", "def run(s=Store()):\n return s.get()\n", 0}, + /* What the signature says it holds unless told otherwise, as a checker reads it. */ + {"parameter default", "def run(s=Store()):\n return s.get()\n", 1}, {"annotated then reassigned", "def run(other, s: Store):\n s = other\n return s.get()\n", 0}, {"self rebound", "class A(Store):\n def run(self, other):\n self = other\n return self.get()\n", 0}, {"static parameter named self", "class A(Store):\n @staticmethod\n def run(self):\n return self.get()\n", 0}, @@ -1385,6 +1377,11 @@ TEST(resolver_python_decorators_keep_identity_only_when_proven) { " if not args:\n return apply\n" " return apply(*args)\n" "def replace(function):\n return 42\n" + "def other():\n return 2\n" + "def replaces(function):\n return other\n" + "def factory(name):\n def apply(function):\n return function\n return apply\n" + "def stubbed(function): ...\n" + "def raises(function):\n raise NotImplementedError\n" "def nothing(function):\n pass\n" "def opaque(function):\n return make(function)\n" "def swallow(function):\n" @@ -1402,19 +1399,25 @@ TEST(resolver_python_decorators_keep_identity_only_when_proven) { int expected; } cases[] = { {"@same", 1}, {"@logged", 1}, {"@tagged('x')", 1}, {"@either", 1}, - {"@either(path='p')", 1}, {"@replace", 0}, {"@nothing", 0}, {"@opaque", 0}, - {"@swallow", 0}, {"@conditional", 0}, {"@generated", 0}, {"@tagged", 0}, - {"@same('x')", 0}, {"@external_deco", 0}, {"@same\n@replace", 0}, {"@same\n@logged", 1}, - {"@wraps_called", 1}, {"@updates", 1}, {"@update_returned", 1}, - {"@hands_on", 1}, {"@hands_on_direct", 1}, {"@hands_to_bad", 0}, - {"@both_arms", 1}, {"@one_arm", 0}, {"@one_arm_wrapped", 0}, + {"@either(path='p')", 1}, {"@same\n@logged", 1}, {"@wraps_called", 1}, {"@updates", 1}, + {"@update_returned", 1}, {"@hands_on", 1}, {"@hands_on_direct", 1}, {"@both_arms", 1}, + /* A decorator it cannot see through still leaves the name on this declaration: + * what runs is dispatch, and flask's @setupmethod and django's @register.filter + * are exactly this. */ + {"@opaque", 1}, {"@swallow", 1}, {"@conditional", 1}, + {"@same('x')", 1}, {"@external_deco", 1}, {"@one_arm", 1}, {"@one_arm_wrapped", 1}, + /* The source proves the name stops holding the function. */ + {"@replace", 0}, {"@replaces", 0}, {"@factory('x')", 1}, {"@tagged", 0}, + {"@nothing", 0}, {"@generated", 0}, {"@same\n@replace", 0}, + {"@hands_to_bad", 1}, {"@stubbed", 1}, {"@raises", 1}, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { char source[768]; snprintf(source, sizeof(source), "from lib.deco import same, logged, tagged, either, replace, nothing, opaque, " "swallow, conditional, generated, wraps_called, updates, update_returned, " - "hands_on, hands_on_direct, hands_to_bad, both_arms, one_arm, one_arm_wrapped\n" + "hands_on, hands_on_direct, hands_to_bad, both_arms, one_arm, one_arm_wrapped, " + "stubbed, raises, replaces, factory\n" "from external import external_deco\n" "%s\ndef target():\n return 1\ndef run():\n return target()\n", cases[i].decorator); @@ -1437,10 +1440,31 @@ TEST(resolver_python_decorators_keep_identity_only_when_proven) { {"main.py", "from functools import total_ordering\n@total_ordering\nclass Base:\n" " def save(self):\n return 1\n" "class Child(Base):\n def run(self):\n return super().save()\n"}); - EXPECT_CALL("base behind an unproven decorator", "main.py", "save", "Method", 0, + /* celery: @Thenable.register class AsyncResult(ResultBase). */ + EXPECT_CALL("base behind an unproven decorator", "main.py", "save", "Method", 1, {"main.py", "from external import wrap\n@wrap\nclass Base:\n" " def save(self):\n return 1\n" "class Child(Base):\n def run(self):\n return super().save()\n"}); + EXPECT_CALL("a base the decorator discards", "main.py", "save", "Method", 0, + {"main.py", "from lib.deco import replace\n@replace\nclass Base:\n" + " def save(self):\n return 1\n" + "class Child(Base):\n def run(self):\n return super().save()\n"}, + decorators); + /* flask: @setupmethod def add_url_rule, called on self from a subclass. */ + EXPECT_CALL("a method behind a decorator it cannot see through", "main.py", "go", "Method", 1, + {"main.py", "from external import setup\nclass Base:\n @setup\n" + " def go(self):\n return 1\n" + "class Child(Base):\n def run(self):\n return self.go()\n"}); + EXPECT_CALL("a method the decorator discards", "main.py", "go", "Method", 0, + {"main.py", "from lib.deco import nothing\nclass Base:\n @nothing\n" + " def go(self):\n return 1\n" + " def run(self):\n return self.go()\n"}, + decorators); + /* celery: kombu's @cached_property, whose value is what the site calls. */ + EXPECT_CALL("a property by its decorator's name is not called", "main.py", "go", "Method", 0, + {"main.py", "from elsewhere import cached_property\nclass A:\n" + " @cached_property\n def go(self):\n return make()\n" + " def run(self):\n return self.go()\n"}); EXPECT_CALL("classmethod subclass binds the function", "main.py", "go", "Method", 1, {"main.py", "from lib.deco import only\nclass A:\n @only\n def go(cls):\n" " return 1\ndef run():\n return A.go()\n"}, @@ -1463,17 +1487,13 @@ TEST(resolver_python_decorators_keep_identity_only_when_proven) { " def read(self):\n return self.value\n"; const python_proof_file_t files[] = {{"main.py", source}}; ASSERT(python_expect("property read", files, 1, "USAGE", "main.py", "value", "Method", 1)); - /* KNOWN GAP, not a contract: a getter and setter share one qualified name, so the - * graph holds one node with the setter's span while a read runs the getter. The - * occurrence abstains as target_declaration_identity_conflict. Tried: nothing in - * the resolver can fix it, since it needs one declaration identity per accessor - * in the definition pass. Flip this to 1 when that exists. */ + /* A getter and setter are one property, and the one node stands for it. */ const char *paired = "class A:\n @property\n def value(self):\n return 1\n" " @value.setter\n def value(self, item):\n pass\n" " def read(self):\n return self.value\n"; const python_proof_file_t pair[] = {{"main.py", paired}}; ASSERT(python_expect("getter and setter share one node", pair, 1, "USAGE", "main.py", "value", - "Method", 0)); + "Method", 1)); const char *argument = "class A:\n @property\n def value(self):\n return 1\n" " def plain(self):\n return 2\n" " def use(self, check):\n" @@ -1496,15 +1516,1263 @@ TEST(resolver_python_decorators_keep_identity_only_when_proven) { ASSERT(python_expect("descriptor read", files, 2, "USAGE", "main.py", "value", "Method", 1)); ASSERT(python_expect("descriptor call", files, 2, "CALLS", "main.py", "value", "Method", 0)); } - /* Unknown decorators on a method stop its lookup too. */ - EXPECT_CALL("unknown method decorator", "main.py", "go", "Method", 0, + /* A decorator whose source nothing shows keeps the declaration, as on a function. */ + EXPECT_CALL("unknown method decorator", "main.py", "go", "Method", 1, {"main.py", "from external import wrap\nclass A:\n @wrap\n def go(self):\n" " return 1\n def run(self):\n return self.go()\n"}); PASS(); } -/* Three suites because each case indexes through both drivers: together they ran past the +/* Five suites because each case indexes through both drivers: together they ran past the * 900 s per-suite cap on Windows, about 500 full indexes in all. */ +/* ── Stated source roots ────────────────────────────────────────────────── + * Forms from click, flask and poetry: a src layout reached from tests and examples, and a + * namespace package kept without __init__.py. Without a manifest nothing states the layout. */ +TEST(resolver_python_stated_source_root_reaches_the_module) { + static const char echo[] = "def echo():\n return 1\n"; + static const char caller[] = "import click\ndef run():\n click.echo()\n"; + { + const python_proof_file_t files[] = { + {"pyproject.toml", "[project]\nname = \"click\"\n"}, + {"src/click/__init__.py", "from .utils import echo as echo\n"}, + {"src/click/utils.py", echo}, + {"tests/test_utils.py", caller}, + {"examples/colors/colors.py", caller}, + {"conftest.py", caller}}; + ASSERT(python_expect_from("a test reaches src by name", files, 6, "tests/test_utils.py", + "CALLS", "src/click/utils.py", "echo", "Function", 1)); + ASSERT(python_expect_from("an example reaches src by name", files, 6, + "examples/colors/colors.py", "CALLS", "src/click/utils.py", + "echo", "Function", 1)); + ASSERT(python_expect_from("a root file reaches src by name", files, 6, "conftest.py", + "CALLS", "src/click/utils.py", "echo", "Function", 1)); + } + { + const python_proof_file_t files[] = {{"src/click/__init__.py", + "from .utils import echo as echo\n"}, + {"src/click/utils.py", echo}, + {"tests/test_utils.py", caller}}; + ASSERT(python_expect_from("no manifest states the layout", files, 3, "tests/test_utils.py", + "CALLS", "src/click/utils.py", "echo", "Function", 0)); + } + { + /* flask/examples: a project of its own, depending on the enclosing one. */ + const python_proof_file_t files[] = { + {"pyproject.toml", "[project]\nname = \"Flask\"\n"}, + {"src/flask/__init__.py", "from .app import Flask as Flask\n"}, + {"src/flask/app.py", "class Flask:\n pass\n"}, + {"examples/celery/pyproject.toml", "[project]\nname = \"flask-example-celery\"\n"}, + {"examples/celery/src/task_app/__init__.py", + "from flask import Flask\nfrom .views import home\ndef create_app():\n" + " home()\n return Flask()\n"}, + {"examples/celery/src/task_app/views.py", echo}, + {"examples/celery/tests/test_app.py", + "from task_app.views import echo\ndef run():\n echo()\n"}}; + ASSERT(python_expect_from("an example reaches the enclosing project", files, 7, + "examples/celery/src/task_app/__init__.py", "CALLS", + "src/flask/app.py", "Flask", "Class", 1)); + ASSERT(python_expect_from("an example's test reaches its own project", files, 7, + "examples/celery/tests/test_app.py", "CALLS", + "examples/celery/src/task_app/views.py", "echo", "Function", 1)); + } + PASS(); +} + +TEST(resolver_python_namespace_package_under_a_source_root) { + /* poetry: src/poetry has no __init__.py and src/poetry/poetry.py shares its name. */ + const python_proof_file_t files[] = { + {"pyproject.toml", "[project]\nname = \"poetry\"\n"}, + {"src/poetry/poetry.py", "class Poetry:\n pass\n"}, + {"src/poetry/config/__init__.py", ""}, + {"src/poetry/config/source.py", "def split_key():\n return 1\n"}, + {"src/poetry/config/config.py", + "from poetry.config.source import split_key\nfrom poetry.poetry import Poetry\n" + "from poetry.core.packages import Package\nfrom config.source import split_key as near\n" + "def merge():\n split_key()\n Poetry()\n Package()\n"}, + {"src/poetry/config/other.py", + "from config.source import split_key\ndef merge():\n split_key()\n"}, + {"src/poetry/__main__.py", + "from poetry.config.source import split_key\ndef main():\n split_key()\n"}, + {"tests/helpers.py", "def helper():\n return 1\n"}, + {"tests/test_config.py", + "from helpers import helper\nfrom poetry.config.source import split_key\n" + "def run():\n helper()\n split_key()\n"}}; + ASSERT(python_expect_from("own namespace package", files, 9, "src/poetry/config/config.py", + "CALLS", "src/poetry/config/source.py", "split_key", "Function", 1)); + ASSERT(python_expect_from("own namespace package, the same-named module", files, 9, + "src/poetry/config/config.py", "CALLS", "src/poetry/poetry.py", + "Poetry", "Class", 1)); + ASSERT(python_expect_from("a package's parent is not on sys.path below a root", files, 9, + "src/poetry/config/other.py", "CALLS", "src/poetry/config/source.py", + "split_key", "Function", 0)); + ASSERT(python_expect_from("a module directly in the namespace package", files, 9, + "src/poetry/__main__.py", "CALLS", "src/poetry/config/source.py", + "split_key", "Function", 1)); + ASSERT(python_expect_from("a test reaches the namespace package", files, 9, + "tests/test_config.py", "CALLS", "src/poetry/config/source.py", + "split_key", "Function", 1)); + ASSERT(python_expect_from("a test's neighbour stays first", files, 9, "tests/test_config.py", + "CALLS", "tests/helpers.py", "helper", "Function", 1)); + PASS(); +} + +/* Each form a backend reads, and one decoy root it must not take from the file's other text. */ +TEST(resolver_python_manifests_state_their_roots) { + static const char go[] = "def go():\n return 1\n"; + static const char caller[] = "import pkg.inner\nfrom pkg.inner import go\ndef run():\n go()\n"; + static const struct { + const char *what, *manifest, *text, *module; + int expected; + } forms[] = { + {"setuptools where", "pyproject.toml", + "[tool.setuptools.packages.find]\nwhere = [\"lib\"]\n", "lib/pkg/inner.py", 1}, + {"setuptools package-dir", "pyproject.toml", + "[tool.setuptools]\npackage-dir = {\"\" = \"lib\"}\n", "lib/pkg/inner.py", 1}, + {"setuptools package-dir table", "pyproject.toml", + "[tool.setuptools.package-dir]\n\"\" = \"lib\"\n", "lib/pkg/inner.py", 1}, + {"poetry from", "pyproject.toml", + "[tool.poetry]\npackages = [\n { include = \"pkg\", from = \"lib\" }, # \"nope\"\n]\n", + "lib/pkg/inner.py", 1}, + {"poetry from as a table", "pyproject.toml", + "[[tool.poetry.packages]]\ninclude = \"pkg\"\nfrom = \"lib\"\n", "lib/pkg/inner.py", 1}, + {"hatch packages", "pyproject.toml", + "[tool.hatch.build.targets.wheel]\npackages = [\"lib/pkg\"]\n", "lib/pkg/inner.py", 1}, + {"pdm package-dir", "pyproject.toml", "[tool.pdm.build]\npackage-dir = \"lib\"\n", + "lib/pkg/inner.py", 1}, + {"maturin python-source", "pyproject.toml", "[tool.maturin]\npython-source = \"lib\"\n", + "lib/pkg/inner.py", 1}, + {"pytest pythonpath", "pyproject.toml", + "[tool.pytest.ini_options]\npythonpath = [\"lib\", \".\"]\n", "lib/pkg/inner.py", 1}, + {"ruff src", "pyproject.toml", "[tool.ruff]\nsrc = [\"lib\"]\n", "lib/pkg/inner.py", 1}, + {"pyright extraPaths", "pyproject.toml", "[tool.pyright]\nextraPaths = [\"lib\"]\n", + "lib/pkg/inner.py", 1}, + {"pytest.ini pythonpath", "pytest.ini", "[pytest]\npythonpath = lib\n", "lib/pkg/inner.py", + 1}, + {"tox.ini pythonpath", "tox.ini", "[pytest]\npythonpath = lib\n", "lib/pkg/inner.py", 1}, + {"setup.cfg package_dir", "setup.cfg", "[options]\npackage_dir =\n = lib\n", + "lib/pkg/inner.py", 1}, + {"setup.cfg where", "setup.cfg", "[options.packages.find]\nwhere = lib\n", + "lib/pkg/inner.py", 1}, + {"setup.py package_dir", "setup.py", "setup(package_dir={\"\": \"lib\"})\n", + "lib/pkg/inner.py", 1}, + {"setup.py find_packages", "setup.py", "setup(packages=find_packages(where='lib'))\n", + "lib/pkg/inner.py", 1}, + {"convention beside setup.py", "setup.py", "setup()\n", "src/pkg/inner.py", 1}, + {"a key elsewhere states nothing", "pyproject.toml", + "[tool.other]\nwhere = [\"lib\"]\n", "lib/pkg/inner.py", 0}, + {"another package's mapping states nothing", "setup.cfg", + "[options]\npackage_dir =\n other = lib\n", "lib/pkg/inner.py", 0}, + {"a commented root states nothing", "pyproject.toml", + "[tool.setuptools.packages.find]\nwhere = [\n # \"lib\"\n \"elsewhere\",\n]\n", + "lib/pkg/inner.py", 0}, + }; + for (size_t i = 0; i < sizeof(forms) / sizeof(forms[0]); i++) { + char init[64]; + snprintf(init, sizeof(init), "%.*s__init__.py", + (int)(strrchr(forms[i].module, '/') - forms[i].module + 1), forms[i].module); + const python_proof_file_t files[] = {{forms[i].manifest, forms[i].text}, + {init, ""}, + {forms[i].module, go}, + {"tests/test_a.py", caller}}; + ASSERT(python_expect_from(forms[i].what, files, 4, "tests/test_a.py", "CALLS", + forms[i].module, "go", "Function", forms[i].expected)); + } + PASS(); +} + +/* pydantic-ai's shape: members of one uv workspace import each other. */ +TEST(resolver_python_workspace_members_see_each_other) { + static const char caller[] = "from beta.inner import go\ndef run():\n go()\n"; + static const char go[] = "def go():\n return 1\n"; + static const char workspace[] = "[tool.uv.workspace]\nmembers = [\"packages/*\", \"extra\"]\n"; + static const char member[] = "[project]\nname = \"member\"\n"; + { + const python_proof_file_t files[] = {{"pyproject.toml", workspace}, + {"packages/alpha/pyproject.toml", member}, + {"packages/alpha/src/alpha/__init__.py", ""}, + {"packages/alpha/src/alpha/user.py", caller}, + {"packages/beta/pyproject.toml", member}, + {"packages/beta/src/beta/__init__.py", ""}, + {"packages/beta/src/beta/inner.py", go}}; + ASSERT(python_expect_from("a sibling member's src layout", files, 7, + "packages/alpha/src/alpha/user.py", "CALLS", + "packages/beta/src/beta/inner.py", "go", "Function", 1)); + } + { + const python_proof_file_t files[] = {{"pyproject.toml", workspace}, + {"extra/pyproject.toml", member}, + {"extra/beta/__init__.py", ""}, + {"extra/beta/inner.py", go}, + {"packages/alpha/pyproject.toml", member}, + {"packages/alpha/alpha/__init__.py", ""}, + {"packages/alpha/alpha/user.py", caller}}; + ASSERT(python_expect_from("an exact member, flat", files, 7, "packages/alpha/alpha/user.py", + "CALLS", "extra/beta/inner.py", "go", "Function", 1)); + } + { + /* Not listed: another project that happens to be in the tree. */ + const python_proof_file_t files[] = {{"pyproject.toml", workspace}, + {"packages/alpha/pyproject.toml", member}, + {"packages/alpha/alpha/__init__.py", ""}, + {"packages/alpha/alpha/user.py", caller}, + {"examples/beta_demo/pyproject.toml", member}, + {"examples/beta_demo/beta/__init__.py", ""}, + {"examples/beta_demo/beta/inner.py", go}}; + ASSERT(python_expect_from("a project outside the members", files, 7, + "packages/alpha/alpha/user.py", "CALLS", + "examples/beta_demo/beta/inner.py", "go", "Function", 0)); + } + { + /* Two members provide the name: nothing says which. */ + const python_proof_file_t files[] = {{"pyproject.toml", workspace}, + {"packages/alpha/pyproject.toml", member}, + {"packages/alpha/alpha/__init__.py", ""}, + {"packages/alpha/alpha/user.py", caller}, + {"packages/one/pyproject.toml", member}, + {"packages/one/beta/__init__.py", ""}, + {"packages/one/beta/inner.py", go}, + {"packages/two/pyproject.toml", member}, + {"packages/two/beta/__init__.py", ""}, + {"packages/two/beta/inner.py", go}}; + ASSERT(python_expect_from("two members of one name", files, 10, + "packages/alpha/alpha/user.py", "CALLS", + "packages/one/beta/inner.py", "go", "Function", 0)); + } + { + /* No workspace declared: siblings stay out of reach. */ + const python_proof_file_t files[] = {{"packages/alpha/pyproject.toml", member}, + {"packages/alpha/alpha/__init__.py", ""}, + {"packages/alpha/alpha/user.py", caller}, + {"packages/beta/pyproject.toml", member}, + {"packages/beta/beta/__init__.py", ""}, + {"packages/beta/beta/inner.py", go}}; + ASSERT(python_expect_from("siblings without a workspace", files, 6, + "packages/alpha/alpha/user.py", "CALLS", + "packages/beta/beta/inner.py", "go", "Function", 0)); + } + PASS(); +} + +TEST(resolver_python_source_root_decoys_abstain) { + static const char go[] = "def go():\n return 1\n"; + static const char manifest[] = "[project]\nname = \"demo\"\n"; + static const char caller[] = "import helper\ndef run():\n helper.go()\n"; + { + /* A src directory that is a package holds src.helper, not helper. */ + const python_proof_file_t files[] = {{"pyproject.toml", manifest}, + {"src/__init__.py", ""}, + {"src/helper.py", go}, + {"tests/test_a.py", caller}}; + ASSERT(python_expect_from("src is itself a package", files, 4, "tests/test_a.py", "CALLS", + "src/helper.py", "go", "Function", 0)); + } + { + /* The standard library comes before an installed distribution. */ + const python_proof_file_t files[] = { + {"pyproject.toml", manifest}, + {"src/logging.py", "def getLogger():\n return 1\n"}, + {"tests/test_a.py", "import logging\ndef run():\n logging.getLogger()\n"}}; + ASSERT(python_expect_from("a standard library name", files, 3, "tests/test_a.py", "CALLS", + "src/logging.py", "getLogger", "Function", 0)); + } + { + const python_proof_file_t files[] = {{"pyproject.toml", manifest}, + {"helper.py", go}, + {"src/helper.py", go}, + {"tests/test_a.py", caller}}; + /* The repository root comes first, as it does for a type checker. */ + ASSERT(python_expect_from("root and src both have it, root", files, 4, "tests/test_a.py", + "CALLS", "helper.py", "go", "Function", 1)); + ASSERT(python_expect_from("root and src both have it, src", files, 4, "tests/test_a.py", + "CALLS", "src/helper.py", "go", "Function", 0)); + } + { + const python_proof_file_t files[] = {{"sdk/pyproject.toml", manifest}, + {"sdk/helper.py", go}, + {"sdk/src/helper.py", go}, + {"sdk/tests/test_a.py", caller}}; + ASSERT(python_expect_from("flat and src both have it, flat", files, 4, + "sdk/tests/test_a.py", "CALLS", "sdk/helper.py", "go", "Function", + 0)); + ASSERT(python_expect_from("flat and src both have it, src", files, 4, "sdk/tests/test_a.py", + "CALLS", "sdk/src/helper.py", "go", "Function", 0)); + } + { + /* A flat project in a subdirectory: its own directory is the root, and its scripts + * still see their neighbours first. */ + const python_proof_file_t files[] = { + {"sdk/python/pyproject.toml", manifest}, + {"sdk/python/mypkg/__init__.py", ""}, + {"sdk/python/mypkg/inner.py", go}, + {"sdk/python/tests/test_a.py", "from mypkg.inner import go\ndef run():\n go()\n"}, + {"sdk/python/scripts/helper.py", "def near():\n return 1\n"}, + {"sdk/python/scripts/run.py", "from helper import near\ndef run():\n near()\n"}, + {"docs/conf.py", "from mypkg.inner import go\ndef run():\n go()\n"}}; + ASSERT(python_expect_from("a flat project in a subdirectory", files, 7, + "sdk/python/tests/test_a.py", "CALLS", + "sdk/python/mypkg/inner.py", "go", "Function", 1)); + ASSERT(python_expect_from("its script sees its neighbour", files, 7, + "sdk/python/scripts/run.py", "CALLS", + "sdk/python/scripts/helper.py", "near", "Function", 1)); + ASSERT(python_expect_from("outside that project it is not on the path", files, 7, + "docs/conf.py", "CALLS", "sdk/python/mypkg/inner.py", "go", + "Function", 0)); + } + { + /* A nearer project's module of that name hides the enclosing project's. */ + const python_proof_file_t files[] = { + {"pyproject.toml", manifest}, + {"src/shared/__init__.py", ""}, + {"src/shared/inner.py", go}, + {"sub/pyproject.toml", manifest}, + {"sub/src/shared/__init__.py", ""}, + {"sub/tests/test_a.py", "from shared.inner import go\ndef run():\n go()\n"}}; + ASSERT(python_expect_from("a nearer project hides the enclosing one", files, 6, + "sub/tests/test_a.py", "CALLS", "src/shared/inner.py", "go", + "Function", 0)); + } + PASS(); +} + +/* ── What changes a module's namespace ──────────────────────────────────── + * Forms from django: a test's `django.__file__ = None` beside `from django import forms`, + * and `from . import signals` in a package's own __init__. */ +TEST(resolver_python_only_a_store_of_that_name_blocks_a_submodule) { + static const python_proof_file_t child = {"pkg/mod.py", "def real():\n return 1\n"}; + static const char main_py[] = "from pkg import mod\ndef run():\n return mod.real()\n"; + static const struct { + const char *patcher; + int expected; + } cases[] = { + /* Handing a module on, and storing some other name, select no other declaration. */ + {"import pkg\ndef patch():\n install(pkg)\n", 1}, + {"import pkg\nsaved = pkg\n", 1}, + {"import pkg\nalias = pkg\nalias.__file__ = None\n", 1}, + {"import pkg\npkg.__file__ = None\n", 1}, + {"import pkg\ndel pkg.__file__\n", 1}, + {"import pkg\nsetattr(pkg, '__file__', None)\n", 1}, + {"import pkg\nwith patch.object(pkg, 'mod', None):\n pass\n", 1}, + {"import pkg.mod\nimport pkg.other\npkg.other.value = 1\n", 1}, + /* The name itself. */ + {"import pkg\npkg.mod = None\n", 0}, + {"import pkg\ndel pkg.mod\n", 0}, + {"import pkg\nsetattr(pkg, 'mod', None)\n", 0}, + {"import pkg\ndelattr(pkg, 'mod')\n", 0}, + {"import pkg\npkg.mod, other = 1, 2\n", 0}, + {"import pkg\nalias = pkg\nalias.mod = None\n", 0}, + {"import pkg\nfirst = pkg\nsecond = first\nsecond.mod = None\n", 0}, + /* Anything could be the name. */ + {"import pkg\nsetattr(pkg, choose(), None)\n", 0}, + {"import pkg\npkg.__dict__['mod'] = None\n", 0}, + {"import pkg\nvars(pkg)['mod'] = None\n", 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + const python_proof_file_t files[] = {{"pkg/__init__.py", ""}, + child, + {"pkg/other.py", "value = 0\n"}, + {"patcher.py", cases[i].patcher}, + {"main.py", main_py}}; + ASSERT(python_expect(cases[i].patcher, files, 5, "CALLS", "pkg/mod.py", "real", "Function", + cases[i].expected)); + } + PASS(); +} + +TEST(resolver_python_writes_through_the_submodule_leave_its_identity) { + /* tests/mail: `del django.conf.settings.X` and mock.patch.object(django.utils.timezone, ...) + * change members; neither changes which module `from django.utils import timezone` is. */ + const python_proof_file_t files[] = { + {"pkg/__init__.py", ""}, + {"pkg/mod.py", "def real():\n return 1\n"}, + {"tests/patcher.py", "import pkg.mod\ndel pkg.mod.settings\npkg.mod.real = None\n"}, + {"main.py", "from pkg import mod\ndef run():\n return mod.real()\n"}}; + ASSERT(python_expect("a store inside the child", files, 4, "CALLS", "pkg/mod.py", "real", + "Function", 1)); + PASS(); +} + +/* click/__init__.py has a __getattr__ for its deprecations, and `from .types import X` above it. */ +TEST(resolver_python_module_getattr_only_answers_what_is_missing) { + static const char go[] = "def real():\n return 1\n"; + static const char user[] = "from pkg import mod\ndef run():\n return mod.real()\n"; + static const char dotted[] = "import pkg\ndef run():\n return pkg.mod.real()\n"; + static const char lazy[] = "def __getattr__(name):\n return load(name)\n"; + { + char init[128]; + snprintf(init, sizeof(init), "from .mod import real as real\n%s", lazy); + const python_proof_file_t files[] = { + {"pkg/__init__.py", init}, {"pkg/mod.py", go}, {"user.py", user}, {"dotted.py", dotted}}; + ASSERT(python_expect_from("a child __init__ loads, by from-import", files, 4, "user.py", + "CALLS", "pkg/mod.py", "real", "Function", 1)); + ASSERT(python_expect_from("a child __init__ loads, by attribute", files, 4, "dotted.py", + "CALLS", "pkg/mod.py", "real", "Function", 1)); + } + { + char init[128]; + snprintf(init, sizeof(init), "from . import mod\n%s", lazy); + const python_proof_file_t files[] = { + {"pkg/__init__.py", init}, {"pkg/mod.py", go}, {"user.py", user}, {"dotted.py", dotted}}; + ASSERT(python_expect_from("a child __init__ binds by from-dot", files, 4, "user.py", "CALLS", + "pkg/mod.py", "real", "Function", 1)); + } + { + const python_proof_file_t files[] = { + {"pkg/__init__.py", lazy}, {"pkg/mod.py", go}, {"user.py", user}, {"dotted.py", dotted}}; + ASSERT(python_expect_from("a child only __getattr__ could answer for", files, 4, "user.py", + "CALLS", "pkg/mod.py", "real", "Function", 0)); + } + PASS(); +} + +TEST(resolver_python_package_importing_its_own_child) { + static const char go[] = "def go():\n return 1\n"; + static const char user[] = "from pkg import mod\ndef run():\n return mod.go()\n"; + static const char direct[] = "import pkg\ndef run():\n return pkg.mod.go()\n"; + static const struct { + const char *init; + int expected; + } cases[] = { + {"from . import mod\n", 1}, + {"from pkg import mod\n", 1}, + {"from . import mod, other\n", 1}, + /* The statement is not the name's only binder. */ + {"mod = None\nfrom . import mod\n", 0}, + {"from . import mod\nmod = None\n", 0}, + {"from .other import mod\n", 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + const python_proof_file_t files[] = {{"pkg/__init__.py", cases[i].init}, + {"pkg/mod.py", go}, + {"pkg/other.py", "mod = 1\n"}, + {"user.py", user}, + {"main.py", direct}}; + ASSERT(python_expect_from(cases[i].init, files, 5, "user.py", "CALLS", "pkg/mod.py", "go", + "Function", cases[i].expected)); + ASSERT(python_expect_from(cases[i].init, files, 5, "main.py", "CALLS", "pkg/mod.py", "go", + "Function", cases[i].expected)); + } + PASS(); +} + +/* ── What a call of a class, and of an unannotated method, gives ────────── + * Forms from django and celery. */ +TEST(resolver_python_special_methods_that_read_through_the_base) { + /* django.utils.functional.LazyObject: answers through super, declared members stand. */ + EXPECT_CALL("__getattribute__ reading through super", "main.py", "go", "Method", 1, + {"main.py", + "class Lazy:\n def __getattribute__(self, name):\n" + " if name == '_wrapped':\n" + " return super().__getattribute__(name)\n" + " value = super().__getattribute__(name)\n return value\n" + " def go(self):\n return 1\n" + " def run(self):\n return self.go()\n"}); + EXPECT_CALL("__getattribute__ answering by itself", "main.py", "go", "Method", 0, + {"main.py", "class Lazy:\n def __getattribute__(self, name):\n" + " return name\n def go(self):\n return 1\n" + " def run(self):\n return self.go()\n"}); + /* django Operation.__new__ allocates; celery chain.__new__ allocates on one path. */ + static const char *allocating[] = { + " self = object.__new__(cls)\n return self\n", + " return super().__new__(cls)\n", + " return super(Made, cls).__new__(cls)\n", + " if args:\n return reduce(args)\n return super().__new__(cls)\n"}; + for (size_t i = 0; i < sizeof(allocating) / sizeof(allocating[0]); i++) { + char source[512]; + snprintf(source, sizeof(source), + "class Made:\n def __new__(cls, *args):\n%s" + " def go(self):\n return 1\n" + "def run():\n made = Made()\n return made.go()\n", + allocating[i]); + const python_proof_file_t file = {"main.py", source}; + ASSERT(python_expect(allocating[i], &file, 1, "CALLS", "main.py", "go", "Method", 1)); + } + EXPECT_CALL("__new__ that never allocates", "main.py", "go", "Method", 0, + {"main.py", "class Made:\n def __new__(cls):\n return other\n" + " def go(self):\n return 1\n" + "def run():\n made = Made()\n return made.go()\n"}); + PASS(); +} + +TEST(resolver_python_returns_follow_the_receiver) { + static const char signature[] = + "class Signature:\n def set(self, **options):\n return self\n" + " def clone(self):\n return copy.copy(self)\n" + " def apply(self):\n return 1\n" + " @classmethod\n def make(cls):\n return cls()\n" + "class Chord(Signature):\n def apply(self):\n return 2\n"; + char source[768]; + /* celery: chord(...).set(...) is a chord, although set is declared on Signature. */ + snprintf(source, sizeof(source), "import copy\n%sdef run():\n return Chord().set().apply()\n", + signature); + { + const python_proof_file_t file = {"main.py", source}; + python_route_release(&python_routes[0]); + python_route_release(&python_routes[1]); + ASSERT(python_expect("return self gives the receiver", &file, 1, "CALLS", "main.py", + "apply", "Method", 1)); + ASSERT_EQ(python_route_edges(&file, 1, "main.py", "CALLS", "main.py", "set", "Method", + false), + 1); + } + { + /* Two files, so the two `apply`s can be told apart. */ + const python_proof_file_t files[] = { + {"sig.py", "class Signature:\n def set(self):\n return self\n" + " def apply(self):\n return 1\n"}, + {"main.py", "from sig import Signature\nclass Chord(Signature):\n" + " def apply(self):\n return 2\n" + "def run():\n return Chord().set().apply()\n"}}; + ASSERT(python_expect("return self selects the subclass's method", files, 2, "CALLS", + "main.py", "apply", "Method", 1)); + ASSERT(python_expect("return self does not select the base's", files, 2, "CALLS", "sig.py", + "apply", "Method", 0)); + } + /* django: BaseExpression.copy returns copy.copy(self). */ + EXPECT_CALL("copy.copy is its argument", "main.py", "only", "Method", 1, + {"main.py", "import copy\nclass A:\n def clone(self):\n" + " return copy.copy(self)\n def only(self):\n" + " return 1\n" + "def run():\n return A().clone().only()\n"}); + EXPECT_CALL("a project copy is not the standard library's", "main.py", "only", "Method", 0, + {"main.py", "from lib import copy\nclass A:\n def clone(self):\n" + " return copy(self)\n def only(self):\n" + " return 1\n" + "def run():\n return A().clone().only()\n"}, + {"lib.py", "def copy(value):\n return 1\n"}); + /* django: ModelAdmin.get_changelist returns the class, get_changelist_instance calls it. */ + EXPECT_CALL("a method returning a class", "main.py", "filters", "Method", 1, + {"main.py", "class Listing:\n def filters(self):\n return 1\n" + "class Admin:\n def kind(self):\n return Listing\n" + " def make(self):\n kind = self.kind()\n" + " return kind()\n" + "def run():\n return Admin().make().filters()\n"}); + /* django.contrib.sitemaps: an unannotated property's value is what it returns. */ + EXPECT_CALL("an unannotated property", "main.py", "page", "Method", 1, + {"main.py", "class Paginator:\n def page(self):\n return 1\n" + "class Sitemap:\n @property\n def paginator(self):\n" + " return Paginator()\n" + " def run(self):\n return self.paginator.page()\n"}); + EXPECT_CALL("a property whose returns disagree", "main.py", "page", "Method", 0, + {"main.py", "class Paginator:\n def page(self):\n return 1\n" + "class Other:\n def page(self):\n return 2\n" + "class Sitemap:\n @property\n def paginator(self):\n" + " if self:\n return Paginator()\n" + " return Other()\n" + " def run(self):\n return self.paginator.page()\n"}); + PASS(); +} + +/* ── Which binder reaches a site ────────────────────────────────────────── + * Forms from django.utils.archive, django.db.models.base and contrib.postgres.constraints. */ +TEST(resolver_python_with_and_except_bind_what_they_say) { + static const char archive[] = + "class Archive:\n def __enter__(self):\n return self\n" + " def __exit__(self, *args):\n return None\n" + " def extract(self):\n return 1\n" + "class Other:\n def extract(self):\n return 2\n" + "class Shut:\n def __enter__(self):\n return Other()\n" + " def __exit__(self, *args):\n return None\n"; + char source[1024]; + snprintf(source, sizeof(source), + "%sdef run():\n with Archive() as archive:\n archive.extract()\n", archive); + { + const python_proof_file_t file = {"main.py", source}; + ASSERT(python_expect("with-as is __enter__'s value", &file, 1, "CALLS", "main.py", + "extract", "Method", 1)); + } + { + /* Both defs of extract share a name: the query counts targets per class below. */ + const python_proof_file_t files[] = { + {"lib.py", archive}, + {"main.py", + "from lib import Shut\ndef run():\n with Shut() as held:\n held.extract()\n"}}; + ASSERT_EQ(python_route_edges(files, 2, "main.py", "CALLS", "lib.py", "extract", "Method", + false), + 1); + ASSERT(python_expect("__enter__ returning another class", files, 2, "CALLS", "lib.py", + "extract", "Method", 1)); + } + EXPECT_CALL("with-as with no __enter__ of its own", "main.py", "extract", "Method", 0, + {"main.py", "from external import Base\nclass Archive(Base):\n" + " def extract(self):\n return 1\n" + "def run():\n with Archive() as archive:\n archive.extract()\n"}); + static const char errors[] = "class Invalid(Exception):\n def update(self):\n return 1\n" + "class Other(Exception):\n def update(self):\n return 2\n"; + { + char text[512]; + snprintf(text, sizeof(text), + "from errors import Invalid, Other\ndef run():\n try:\n pass\n" + " except Invalid as e:\n e.update()\n"); + const python_proof_file_t files[] = {{"errors.py", errors}, {"main.py", text}}; + ASSERT(python_expect("except-as is the class", files, 2, "CALLS", "errors.py", "update", + "Method", 1)); + } + { + /* base.py: one name across two clauses; each clause sees its own class. */ + const python_proof_file_t files[] = { + {"errors.py", "class Invalid(Exception):\n def only(self):\n return 1\n" + "class Other(Exception):\n def rest(self):\n return 2\n"}, + {"main.py", + "from errors import Invalid, Other\ndef run():\n try:\n pass\n" + " except Invalid as e:\n e.only()\n except Other as e:\n e.rest()\n"}}; + ASSERT(python_expect("first clause", files, 2, "CALLS", "errors.py", "only", "Method", 1)); + ASSERT(python_expect("second clause", files, 2, "CALLS", "errors.py", "rest", "Method", 1)); + } + { + const python_proof_file_t files[] = { + {"errors.py", errors}, + {"main.py", "from errors import Invalid, Other\ndef run():\n try:\n pass\n" + " except (Invalid, Other) as e:\n e.update()\n"}}; + ASSERT(python_expect("a tuple of classes says neither", files, 2, "CALLS", "errors.py", + "update", "Method", 0)); + } + PASS(); +} + +TEST(resolver_python_the_binder_every_path_passes_last) { + static const char classes[] = "class A:\n def go(self):\n return 1\n" + " def clone(self):\n return self\n" + "class B:\n def go(self):\n return 2\n"; + static const struct { + const char *what, *body; + int a, b; + } cases[] = { + {"straight line", " x = A()\n x = B()\n x.go()\n", 0, 1}, + {"before the rebinding", " x = A()\n x.go()\n x = B()\n", 1, 0}, + /* contrib/postgres/constraints.py: the loop variable rebound by a real class. */ + {"rebinding a loop variable", + " for x in items:\n x = A()\n x.go()\n", 1, 0}, + {"a nested use after the binder", " x = A()\n if c:\n x.go()\n x = B()\n", 1, 0}, + {"inside an arm", " x = B()\n if c:\n x = A()\n x.go()\n", 1, 0}, + {"after an arm, either could reach", " x = B()\n if c:\n x = A()\n x.go()\n", + 0, 0}, + {"the right side reads the earlier binder", " x = A()\n x = x.clone()\n x.go()\n", + 1, 0}, + /* A later binder comes back round through the loop. */ + {"a loop that rebinds later", " x = A()\n for i in items:\n x.go()\n x = B()\n", + 0, 0}, + {"a loop that rebinds elsewhere", " x = A()\n for i in items:\n x.go()\n x = B()\n", + 1, 0}, + {"a while that rebinds later", " x = A()\n while c:\n x.go()\n x = B()\n", + 0, 0}, + {"binder and use in one iteration", " for i in items:\n x = A()\n x.go()\n x = B()\n", + 1, 0}, + {"another binder form", " x = A()\n for x in items:\n x.go()\n", 0, 0}, + {"a None between", " x = A()\n x = None\n x.go()\n", 0, 0}, + {"something unfollowable between", " x = A()\n x = pick()\n x.go()\n", 0, 0}, + {"a parameter, rebound later", " x.go()\n x = A()\n", 0, 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char source[512]; + snprintf(source, sizeof(source), "from lib import A, B\ndef run(x, items, c):\n%s", + cases[i].body); + const python_proof_file_t files[] = {{"lib.py", classes}, {"main.py", source}}; + char label[96]; + snprintf(label, sizeof(label), "%s, A", cases[i].what); + python_route_release(&python_routes[0]); + python_route_release(&python_routes[1]); + /* The two classes share go: their nodes differ by start line, so count by class. */ + int a = python_route_edges(files, 2, "main.py", "CALLS", "lib.py", "go", "Method", false); + ASSERT_EQ(a, cases[i].a + cases[i].b); + ASSERT(python_expect(label, files, 2, "CALLS", "lib.py", "go", "Method", + cases[i].a + cases[i].b)); + } + /* Which class, for the cases where only one is right. */ + EXPECT_CALL("straight line picks the last", "lib.py", "only_b", "Method", 1, + {"lib.py", "class A:\n def only_a(self):\n return 1\n" + "class B:\n def only_b(self):\n return 2\n"}, + {"main.py", "from lib import A, B\ndef run():\n x = A()\n x = B()\n" + " x.only_b()\n"}); + EXPECT_CALL("and not the one before", "lib.py", "only_a", "Method", 0, + {"lib.py", "class A:\n def only_a(self):\n return 1\n" + "class B:\n def only_b(self):\n return 2\n"}, + {"main.py", "from lib import A, B\ndef run():\n x = A()\n x = B()\n" + " x.only_a()\n"}); + PASS(); +} + +TEST(resolver_python_names_a_module_states_only_for_checkers) { + /* celery/__init__.py: lazy at run time, stated under TYPE_CHECKING. */ + static const char canvas[] = "class chain:\n def link(self):\n return 1\n"; + static const char user[] = "from pkg import chain\ndef run():\n s = chain()\n s.link()\n"; + { + const python_proof_file_t files[] = { + {"pkg/__init__.py", "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n" + " from pkg.canvas import chain\n" + "def __getattr__(name):\n return load(name)\n"}, + {"pkg/canvas.py", canvas}, + {"main.py", user}}; + ASSERT(python_expect("the class", files, 3, "CALLS", "pkg/canvas.py", "chain", "Class", 1)); + ASSERT(python_expect("its method", files, 3, "CALLS", "pkg/canvas.py", "link", "Method", 1)); + } + { + const python_proof_file_t files[] = { + {"pkg/__init__.py", "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n" + " from pkg.canvas import chain\nelse:\n chain = None\n"}, + {"pkg/canvas.py", canvas}, + {"main.py", user}}; + ASSERT(python_expect("a runtime binder competes", files, 3, "CALLS", "pkg/canvas.py", + "chain", "Class", 0)); + } + { + /* celery: `local.recreate_module(__name__, ...)` builds the module at run time. */ + const python_proof_file_t handed[] = { + {"pkg/__init__.py", "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n" + " from pkg.canvas import chain\n" + "import local\nlocal.recreate_module(__name__, by={})\n"}, + {"pkg/canvas.py", canvas}, + {"main.py", user}}; + ASSERT(python_expect("a module that hands on its own name", handed, 3, "CALLS", + "pkg/canvas.py", "chain", "Class", 1)); + /* Nothing provides the name at run time: no __getattr__, no binder. */ + const python_proof_file_t files[] = { + {"pkg/__init__.py", "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n" + " from pkg.canvas import chain\n"}, + {"pkg/canvas.py", canvas}, + {"main.py", user}}; + ASSERT(python_expect("a closed module's checker-only name", files, 3, "CALLS", + "pkg/canvas.py", "chain", "Class", 0)); + } + { + const python_proof_file_t files[] = { + {"pkg/__init__.py", "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n" + " from pkg.canvas import *\n"}, + {"pkg/canvas.py", canvas}, + {"main.py", user}}; + ASSERT(python_expect("a closed module's checker-only star", files, 3, "CALLS", + "pkg/canvas.py", "chain", "Class", 0)); + } + { + /* A lazy package states a whole module's names by a star under TYPE_CHECKING. */ + const python_proof_file_t files[] = { + {"pkg/__init__.py", "import typing\nif typing.TYPE_CHECKING:\n from .canvas import *\n" + "def __getattr__(name):\n return load(name)\n"}, + {"pkg/canvas.py", "__all__ = ['chain']\nclass chain:\n def link(self):\n" + " return 1\nclass hidden:\n pass\n"}, + {"main.py", "from pkg import chain, hidden\ndef run():\n hidden()\n s = chain()\n" + " s.link()\n"}}; + ASSERT(python_expect("a star under TYPE_CHECKING", files, 3, "CALLS", "pkg/canvas.py", "link", + "Method", 1)); + ASSERT(python_expect("what that star does not export", files, 3, "CALLS", "pkg/canvas.py", + "hidden", "Class", 0)); + } + { + const python_proof_file_t files[] = { + {"pkg/__init__.py", "import os\nif os.environ:\n from .canvas import *\n" + "else:\n from .other import *\n"}, + {"pkg/canvas.py", "__all__ = ['chain']\nclass chain:\n pass\n"}, + {"pkg/other.py", "__all__ = ['chain']\nclass chain:\n pass\n"}, + {"main.py", "from pkg import chain\ndef run():\n chain()\n"}}; + ASSERT(python_expect("two stars a condition chooses between", files, 4, "CALLS", + "pkg/canvas.py", "chain", "Class", 0)); + ASSERT(python_expect("the other of those stars", files, 4, "CALLS", "pkg/other.py", "chain", + "Class", 0)); + } + /* A class body's `X = X` reads X from outside: celery's ResultConsumer. */ + EXPECT_CALL("a class attribute named as what it holds", "main.py", "go", "Method", 1, + {"main.py", "class Consumer:\n def go(self):\n return 1\n" + "class Backend:\n Consumer = Consumer\n def make(self):\n" + " return self.Consumer()\n" + "def run():\n return Backend().make().go()\n"}); + /* tests/migrations/test_writer.py: exec's assignments go to its third argument. */ + EXPECT_CALL("exec with globals() and its own locals", "main.py", "local", "Function", 1, + {"main.py", "def local():\n return 1\ndef safe(code):\n d = {}\n" + " exec(code, globals(), d)\ndef run():\n return local()\n"}); + EXPECT_CALL("exec into globals() itself", "main.py", "local", "Function", 0, + {"main.py", "def local():\n return 1\ndef safe(code):\n" + " exec(code, globals())\ndef run():\n return local()\n"}); + /* A computed key names no binding the audit saw. */ + EXPECT_CALL("a computed globals() store", "main.py", "local", "Function", 1, + {"main.py", "def local():\n return 1\nglobals()[key()] = None\n" + "def run():\n return local()\n"}); + EXPECT_CALL("a literal globals() store", "main.py", "local", "Function", 0, + {"main.py", "def local():\n return 1\nglobals()['local'] = None\n" + "def run():\n return local()\n"}); + { + /* What the module lacks is unknowable after one, so a submodule is unproven. */ + const python_proof_file_t files[] = { + {"pkg/__init__.py", "globals()[key()] = None\n"}, + {"pkg/mod.py", "def real():\n return 1\n"}, + {"main.py", "from pkg import mod\ndef run():\n return mod.real()\n"}}; + ASSERT(python_expect("absence after a computed store", files, 3, "CALLS", "pkg/mod.py", + "real", "Function", 0)); + } + /* test_geos_mutation.py: a read-only method of the view. */ + EXPECT_CALL("vars().items() reads", "main.py", "local", "Function", 1, + {"main.py", "def local():\n return 1\nnames = list(vars().items())\n" + "def run():\n return local()\n"}); + PASS(); +} + +/* click's @t.overload def prompt, and Command.main. */ +TEST(resolver_python_overload_stubs_are_one_declaration) { + static const char lib[] = + "import typing as t\nfrom typing import overload\ndef helper():\n return 1\n" + "@t.overload\ndef prompt(text: str) -> str: ...\n" + "@t.overload\ndef prompt(text: int) -> int: ...\n" + "def prompt(text):\n return helper()\n" + "class Command:\n @overload\n def main(self, a: str) -> str: ...\n" + " @overload\n def main(self, a: int) -> int: ...\n" + " def main(self, a):\n return self.invoke(a)\n" + " def invoke(self, a):\n return helper()\n"; + static const char user[] = "from lib import prompt, Command\ndef run():\n prompt('x')\n" + " Command().main('x')\n"; + const python_proof_file_t files[] = {{"lib.py", lib}, {"main.py", user}}; + ASSERT(python_expect("an overloaded function", files, 2, "CALLS", "lib.py", "prompt", + "Function", 1)); + ASSERT(python_expect("an overloaded method", files, 2, "CALLS", "lib.py", "main", "Method", 1)); + ASSERT(python_expect_from("from an overloaded function's body", files, 2, "lib.py", "CALLS", + "lib.py", "helper", "Function", 2)); + ASSERT(python_expect_from("from an overloaded method's body", files, 2, "lib.py", "CALLS", + "lib.py", "invoke", "Method", 1)); + /* A project decorator of that name proves nothing: two defs, the later one rebinds. */ + EXPECT_CALL("a project decorator named overload", "main.py", "go", "Function", 0, + {"main.py", "from project import overload\n@overload\ndef go():\n return 1\n" + "def go():\n return 2\ndef run():\n return go()\n"}, + {"project.py", "def overload(function):\n return function\n"}); + PASS(); +} + +TEST(resolver_python_loading_a_child_the_parent_imported) { + /* django.db.models: `from django.db.models import signals` in its own __init__, and other + * files load django.db.models.signals. Both name the one module. */ + const python_proof_file_t files[] = { + {"pkg/__init__.py", "from pkg import signals\n"}, + {"pkg/signals.py", "class Signal:\n def send(self):\n return 1\n" + "pre_delete = Signal()\n"}, + {"loader.py", "from pkg.signals import pre_delete\n"}, + {"main.py", "from pkg import signals\ndef run():\n return signals.pre_delete.send()\n"}}; + ASSERT(python_expect("through the parent", files, 4, "CALLS", "pkg/signals.py", "send", + "Method", 1)); + /* Under another name the parent's binding is not the module's own attribute. */ + const python_proof_file_t renamed[] = { + {"pkg/__init__.py", "from pkg.other import value as signals\n"}, + {"pkg/other.py", "value = 1\n"}, + {"pkg/signals.py", "class Signal:\n def send(self):\n return 1\n" + "pre_delete = Signal()\n"}, + {"loader.py", "from pkg.signals import pre_delete\n"}, + {"main.py", "from pkg import signals\ndef run():\n return signals.pre_delete.send()\n"}}; + ASSERT(python_expect("a value of that name competes", renamed, 5, "CALLS", "pkg/signals.py", + "send", "Method", 0)); + PASS(); +} + +/* ── What annotations and isinstance say ────────────────────────────────── + * Forms from click (dict[str, _Option], -> list[Parameter]), httpx (self: T -> T, async with), + * poetry (isinstance on a loop variable) and django (isinstance on self.rhs). */ +TEST(resolver_python_annotated_containers_hold_their_element) { + static const char lib[] = + "from __future__ import annotations\nimport typing as t\n" + "from typing import Iterator, Optional, Self, TypeVar\n" + "from contextlib import contextmanager\nT = TypeVar('T', bound='Client')\n" + "class Param:\n def clean(self):\n return 1\n" + "class Option:\n def process(self):\n return 1\n" + "class Response:\n def read(self):\n return 1\n" + "class Gen(t.Generic[T]):\n def own(self):\n return 1\n" + "class Client:\n def __init__(self):\n self._opts: dict[str, Option] = {}\n" + " self._params: list[Param] = []\n" + " def __enter__(self: T) -> T:\n return self\n" + " def chain(self) -> Self:\n return self\n" + " def only(self):\n return 1\n" + " def params(self) -> list[Param]:\n return self._params\n" + " def first(self) -> Optional[Param]:\n return None\n" + " @contextmanager\n def stream(self) -> Iterator[Response]:\n" + " yield Response()\n" + "class Sub(Client):\n def only(self):\n return 2\n"; + static const struct { + const char *what, *body, *name; + int expected; + } cases[] = { + {"for over a returned list", + "def run(c: Client):\n for p in c.params():\n p.clean()\n", "clean", 1}, + {"for over an annotated attribute", + "def run(c: Client):\n for p in c._params:\n p.clean()\n", "clean", 1}, + {"a comprehension", "def run(c: Client):\n return [p.clean() for p in c._params]\n", + "clean", 1}, + {"sorted keeps what it was given", + "def run(c: Client):\n for p in sorted(c._params, key=None):\n p.clean()\n", + "clean", 1}, + {"reversed keeps what it was given", + "def run(c: Client):\n for p in reversed(c._params):\n p.clean()\n", "clean", 1}, + {"a project function named sorted says nothing", + "from other import sorted\ndef run(c: Client):\n for p in sorted(c._params):\n" + " p.clean()\n", + "clean", 0}, + {"a subscript of a list annotation", + "def run(items: list[Param]):\n return items[0].clean()\n", "clean", 1}, + {"a slice stays a list", + "def run(items: list[Param]):\n for p in items[1:]:\n p.clean()\n", "clean", 1}, + {"a dict value by key", "def run(c: Client, k):\n c._opts[k].process()\n", "process", 1}, + {"a dict value by get", "def run(c: Client, k):\n c._opts.get(k).process()\n", "process", + 1}, + {"dict values", "def run(c: Client):\n for o in c._opts.values():\n o.process()\n", + "process", 1}, + {"iterating a dict yields keys", + "def run(c: Client):\n for o in c._opts:\n o.process()\n", "process", 0}, + {"items, the value", + "def run(c: Client):\n for k, o in c._opts.items():\n o.process()\n", "process", + 1}, + {"items of a class that has its own", + "class Mine:\n def items(self):\n return [(1, 2)]\n" + " def values(self) -> list[Param]:\n return []\n" + "def run(m: Mine):\n for k, p in m.items():\n p.clean()\n", + "clean", 0}, + {"items, the key says nothing", + "def run(c: Client):\n for o, k in c._opts.items():\n o.process()\n", "process", + 0}, + {"enumerate, the element", + "def run(c: Client):\n for i, p in enumerate(c._params):\n p.clean()\n", + "clean", 1}, + {"enumerate, the count says nothing", + "def run(c: Client):\n for p, i in enumerate(c._params):\n p.clean()\n", + "clean", 0}, + {"zip, by position", + "def run(c: Client, xs):\n for x, p in zip(xs, c._params):\n p.clean()\n", + "clean", 1}, + {"zip, the other position", + "def run(c: Client, xs):\n for p, x in zip(xs, c._params):\n p.clean()\n", + "clean", 0}, + {"a project enumerate says nothing", + "from other import enumerate\ndef run(c: Client):\n" + " for i, p in enumerate(c._params):\n p.clean()\n", + "clean", 0}, + {"a walrus", "def run(c: Client):\n if (p := c.first()):\n p.clean()\n", "clean", + 1}, + {"a typing alias", + "import typing as t\ndef run(items: t.Sequence[Param]):\n for p in items:\n" + " p.clean()\n", + "clean", 1}, + {"typing.cast", "import typing as t\ndef run(v):\n t.cast('Param', v).clean()\n", "clean", + 1}, + {"a project cast is not typing's", + "from other import cast\ndef run(v):\n cast('Param', v).clean()\n", "clean", 0}, + {"type[X] is the class", + "def run(kind: type[Param]):\n return kind().clean()\n", "clean", 1}, + {"a generic of the project's own", + "def run(g: Gen[int]):\n return g.own()\n", "own", 1}, + {"a contextmanager's yielded type", + "def run(c: Client):\n with c.stream() as r:\n r.read()\n", "read", 1}, + /* Nothing here says which class holds the name. */ + {"an unannotated parameter", "def run(items):\n for p in items:\n p.clean()\n", + "clean", 0}, + {"nested containers are not carried", + "def run(items: list[list[Param]]):\n for p in items:\n p.clean()\n", "clean", 0}, + {"a tuple of two types is not a sequence of one", + "def run(items: tuple[Param, Option]):\n return items[0].clean()\n", "clean", 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char source[640]; + snprintf(source, sizeof(source), "from lib import Client, Param, Option, Gen\n%s", + cases[i].body); + const python_proof_file_t files[] = {{"lib.py", lib}, + {"other.py", "def sorted(x):\n return 1\n" + "def cast(x, y):\n return 1\n" + "def enumerate(x):\n return 1\n"}, + {"main.py", source}}; + ASSERT(python_expect(cases[i].what, files, 3, "CALLS", "lib.py", cases[i].name, "Method", + cases[i].expected)); + } + /* A method annotated to return its receiver gives the receiver's class. */ + { + const python_proof_file_t files[] = { + {"lib.py", lib}, + {"main.py", "from lib import Sub\ndef run(c: Sub):\n c.chain().only()\n" + " with c as held:\n held.only()\n"}}; + int found = python_route_edges(files, 2, "main.py", "CALLS", "lib.py", "only", "Method", + false); + ASSERT_EQ(found, 1); + } + PASS(); +} + +TEST(resolver_python_isinstance_narrows_where_it_holds) { + static const char classes[] = "class Stmt:\n def rename(self):\n return 1\n" + "class Other:\n def rename(self):\n return 2\n" + "def make():\n return None\n"; + static const struct { + const char *what, *body; + int expected; + } cases[] = { + {"a parameter", "def run(x):\n if isinstance(x, Stmt):\n x.rename()\n", 1}, + {"a loop variable", + "def run(items):\n for x in items:\n if isinstance(x, Stmt):\n" + " x.rename()\n", + 1}, + {"self's attribute", + "class H:\n def run(self):\n if isinstance(self.rhs, Stmt):\n" + " self.rhs.rename()\n", + 1}, + {"an early exit", "def run(x):\n if not isinstance(x, Stmt):\n return None\n" + " x.rename()\n", 1}, + {"an early raise", "def run(x):\n if not isinstance(x, Stmt):\n raise ValueError()\n" + " x.rename()\n", 1}, + {"an assert", "def run(x):\n assert isinstance(x, Stmt)\n x.rename()\n", 1}, + {"a conjunct", "def run(x, c):\n if c and isinstance(x, Stmt):\n x.rename()\n", 1}, + {"after the other conjunct", + "def run(x, c):\n if isinstance(x, Stmt) and x.rename():\n pass\n", 1}, + {"elif", "def run(x):\n if x:\n pass\n elif isinstance(x, Stmt):\n" + " x.rename()\n", 1}, + /* Where it does not hold. */ + {"after the if", "def run(x):\n if isinstance(x, Stmt):\n pass\n x.rename()\n", + 0}, + {"the else", "def run(x):\n if isinstance(x, Stmt):\n pass\n else:\n" + " x.rename()\n", 0}, + {"a not without the exit", + "def run(x):\n if not isinstance(x, Stmt):\n print(x)\n x.rename()\n", 0}, + {"a tuple of classes", + "def run(x):\n if isinstance(x, (Stmt, Other)):\n x.rename()\n", 0}, + {"rebound inside", "def run(x):\n if isinstance(x, Stmt):\n x = make()\n" + " x.rename()\n", 0}, + {"stored inside", + "class H:\n def run(self):\n if isinstance(self.rhs, Stmt):\n" + " self.rhs = make()\n self.rhs.rename()\n", + 0}, + {"in a nested function", + "def run(x):\n if isinstance(x, Stmt):\n def later():\n x.rename()\n", + 0}, + {"a shadowed isinstance", + "def run(x, isinstance):\n if isinstance(x, Stmt):\n x.rename()\n", 0}, + {"a computed value", + "def run(x):\n if isinstance(x.pop(), Stmt):\n x.pop().rename()\n", 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char source[512]; + snprintf(source, sizeof(source), "from lib import Stmt, Other, make\n%s", cases[i].body); + const python_proof_file_t files[] = {{"lib.py", classes}, {"main.py", source}}; + ASSERT(python_expect(cases[i].what, files, 2, "CALLS", "lib.py", "rename", "Method", + cases[i].expected)); + } + PASS(); +} + +/* ── The smaller shapes django's tree has ───────────────────────────────── */ +TEST(resolver_python_class_and_module_shapes) { + /* A class stated inside a class, read through either. */ + EXPECT_CALL("a nested class by its owner", "main.py", "go", "Method", 1, + {"main.py", "class Tests:\n class Form:\n def go(self):\n" + " return 1\n" + " def run(self):\n form = Tests.Form()\n" + " return form.go()\n"}); + EXPECT_CALL("a nested class by self", "main.py", "go", "Method", 1, + {"main.py", "class Tests:\n class Form:\n def go(self):\n" + " return 1\n" + " def run(self):\n return self.Form().go()\n"}); + /* geos: ListMixin stores self._get_single_internal only when no subclass declares it. */ + EXPECT_CALL("a store in a base beyond the declaring class", "main.py", "go", "Method", 1, + {"main.py", "class Mixin:\n def __init__(self):\n self.go = make()\n" + "class Child(Mixin):\n def go(self):\n return 1\n" + " def run(self):\n return self.go()\n"}); + EXPECT_CALL("a store in the declaring class", "main.py", "go", "Method", 0, + {"main.py", "class Child:\n def __init__(self):\n self.go = make()\n" + " def go(self):\n return 1\n" + " def run(self):\n return self.go()\n"}); + /* point.py: a property's setter is the same property. */ + { + const python_proof_file_t files[] = { + {"main.py", "def helper():\n return 1\nclass P:\n @property\n def x(self):\n" + " return helper()\n @x.setter\n def x(self, value):\n" + " helper()\n"}}; + ASSERT(python_expect_from("calls from a property with a setter", files, 1, "main.py", "CALLS", + "main.py", "helper", "Function", 1)); + } + /* tests/backends: a subclass's store selects over its base's, as a method would. */ + { + const python_proof_file_t files[] = { + {"lib.py", "class Table:\n def only_table(self):\n return 1\n" + "class Columns:\n def only_columns(self):\n return 2\n"}, + {"main.py", "from lib import Table, Columns\nclass Base:\n def setUp(self):\n" + " self.reference = Table()\n" + "class Sub(Base):\n def setUp(self):\n" + " self.reference = Columns()\n" + " def run(self):\n return self.reference.only_columns()\n"}}; + ASSERT(python_expect("the nearer store selects", files, 2, "CALLS", "lib.py", "only_columns", + "Method", 1)); + } + /* click: self.type: ParamType = ... in Parameter, self.type = convert(...) in Option. */ + EXPECT_CALL("an annotation holds what a subclass stores", "main.py", "go", "Method", 1, + {"main.py", "class Kind:\n def go(self):\n return 1\n" + "class Base:\n def __init__(self):\n" + " self.kind: Kind = make()\n" + "class Sub(Base):\n def set(self):\n self.kind = other()\n" + " def run(self):\n return self.kind.go()\n"}); + /* tests: cls.a = Article() in setUpTestData, read back as cls.a. */ + EXPECT_CALL("a store through cls read through cls", "main.py", "go", "Method", 1, + {"main.py", "class Article:\n def go(self):\n return 1\n" + "class T:\n @classmethod\n def setup(cls):\n" + " cls.a = Article()\n cls.a.go()\n"}); + /* `self.go = make()` rebinds an instance's name, and the class keeps its own. */ + EXPECT_CALL("a declared method through its class beside an instance store", "main.py", "go", + "Method", 1, + {"main.py", "class T:\n def __init__(self):\n self.go = make()\n" + " def go(self):\n return 1\n" + "def run(t):\n return T.go(t)\n"}); + EXPECT_CALL("a store through cls over the class's None", "main.py", "go", "Method", 1, + {"main.py", "class Article:\n def go(self):\n return 1\n" + "class T:\n a = None\n @classmethod\n def setup(cls):\n" + " cls.a = Article()\n cls.a.go()\n"}); + EXPECT_CALL("a store through cls beside a store of something else", "main.py", "go", "Method", + 0, + {"main.py", "class Article:\n def go(self):\n return 1\n" + "class T:\n @classmethod\n def setup(cls, other):\n" + " cls.a = Article()\n cls.a = other\n" + " cls.a.go()\n"}); + /* `app.conf = Mock()` says what app.conf holds from there on. */ + EXPECT_CALL("a store through a receiver before the read", "main.py", "go", "Method", 0, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class T:\n x = A()\n" + "def run(t: T):\n t.x = make()\n t.x.go()\n"}); + EXPECT_CALL("a store of what it was declared to hold", "main.py", "go", "Method", 1, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class T:\n x = A()\n" + "def run(t: T):\n t.x = A()\n t.x.go()\n"}); + EXPECT_CALL("a store of something else", "main.py", "go", "Method", 0, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class B:\n def go(self):\n return 2\n" + "class T:\n x = A()\n" + "def run(t: T):\n t.x = B()\n t.x.go()\n"}); + { + /* The member itself is still the member: `r.content = x` then reading r.content. */ + const python_proof_file_t files[] = { + {"main.py", "class R:\n @property\n def content(self):\n return 1\n" + " @content.setter\n def content(self, value):\n pass\n" + "def run(r: R):\n r.content = 2\n return r.content\n"}}; + ASSERT(python_expect("a read of the member just stored", files, 1, "USAGE", "main.py", + "content", "Method", 1)); + } + /* A class-body value whose class has __get__: the read runs that. */ + EXPECT_CALL("a descriptor's own method", "main.py", "go", "Method", 0, + {"main.py", "class D:\n def __get__(self, i, o):\n return make()\n" + " def go(self):\n return 1\n" + "class T:\n x = D()\n" + " def run(self):\n self.x.go()\n"}); + EXPECT_CALL("what a descriptor's __get__ returns", "main.py", "go", "Method", 1, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class D:\n def __get__(self, i, o):\n return A()\n" + "class T:\n x = D()\n" + " def run(self):\n self.x.go()\n"}); + EXPECT_CALL("a store through a receiver after the read", "main.py", "go", "Method", 1, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class T:\n x = A()\n" + "def run(t: T):\n t.x.go()\n t.x = make()\n"}); + EXPECT_CALL("a store through that spelling in another function", "main.py", "go", "Method", 1, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class T:\n x = A()\n" + "def other(t: T):\n t.x = make()\n" + "def run(t: T):\n t.x.go()\n"}); + EXPECT_CALL("a store at module level before the function", "main.py", "go", "Method", 0, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class T:\n x = A()\n" + "t = T()\nt.x = make()\n" + "def run():\n t.x.go()\n"}); + EXPECT_CALL("a store past the first attribute of self", "main.py", "go", "Method", 0, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class H:\n y = A()\n" + "class T:\n def __init__(self):\n self.x = H()\n" + " def run(self):\n self.x.y = make()\n" + " self.x.y.go()\n"}); + EXPECT_CALL("the value of a store reads what was there", "main.py", "go", "Method", 1, + {"main.py", "class A:\n def go(self):\n return 1\n" + "class T:\n x = A()\n" + "def run(t: T):\n t.x = t.x.go()\n"}); + /* QuerySet.as_manager: `as_manager = classmethod(as_manager)`, the decorator spelled out. */ + EXPECT_CALL("a classmethod spelled out", "main.py", "go", "Method", 1, + {"main.py", "class T:\n def go(cls):\n return 1\n" + " go = classmethod(go)\n" + "class S(T):\n pass\n" + "def run():\n return S.go()\n"}); + EXPECT_CALL("a staticmethod spelled out", "main.py", "go", "Method", 1, + {"main.py", "class T:\n def go():\n return 1\n" + " go = staticmethod(go)\n" + " def run(self):\n return self.go()\n"}); + EXPECT_CALL("a project's own classmethod", "main.py", "go", "Method", 0, + {"main.py", "from other import classmethod\n" + "class T:\n def go(cls):\n return 1\n" + " go = classmethod(go)\n" + "def run():\n return T.go()\n"}, + {"other.py", "def classmethod(f):\n return f\n"}); + EXPECT_CALL("a classmethod of something else", "main.py", "go", "Method", 0, + {"main.py", "def other(cls):\n return 1\n" + "class T:\n def go(cls):\n return 1\n" + " go = classmethod(other)\n" + "def run():\n return T.go()\n"}); + /* `def f(self, Request=Request)`: what the signature says it holds. */ + EXPECT_CALL("a parameter defaulting to a class", "main.py", "go", "Method", 1, + {"main.py", "class Request:\n def go(self):\n return 1\n" + "def run(Request=Request):\n return Request().go()\n"}); + EXPECT_CALL("a parameter defaulting to an instance", "main.py", "go", "Method", 1, + {"main.py", "class Apps:\n def go(self):\n return 1\nglobal_apps = Apps()\n" + "def run(apps=global_apps):\n return apps.go()\n"}); + EXPECT_CALL("a parameter with no default says nothing", "main.py", "go", "Method", 0, + {"main.py", "class Request:\n def go(self):\n return 1\n" + "def run(Request):\n return Request().go()\n"}); + /* `questioner or Questioner()` for a parameter defaulting to None. */ + EXPECT_CALL("None or a class", "main.py", "go", "Method", 1, + {"main.py", "class Q:\n def go(self):\n return 1\n" + "class A:\n def __init__(self, q=None):\n" + " self.q = q or Q()\n" + " def run(self):\n return self.q.go()\n"}); + EXPECT_CALL("a required parameter or a class", "main.py", "go", "Method", 0, + {"main.py", "class Q:\n def go(self):\n return 1\n" + "class A:\n def __init__(self, q):\n" + " self.q = q or Q()\n" + " def run(self):\n return self.q.go()\n"}); + /* `A() if c else None`. */ + EXPECT_CALL("a value or None", "main.py", "go", "Method", 1, + {"main.py", "class A:\n def go(self):\n return 1\n" + "def run(c):\n x = A() if c else None\n return x.go()\n"}); + EXPECT_CALL("two values", "main.py", "go", "Method", 0, + {"main.py", "class A:\n def go(self):\n return 1\n" + "def run(c, y):\n x = A() if c else y\n return x.go()\n"}); + /* django.contrib.gis.db.models: a star of a module whose __all__ is computed. */ + { + const python_proof_file_t files[] = { + {"base/__init__.py", "from base.parts import __all__ as parts_all\n" + "from base.parts import *\n" + "__all__ = parts_all\n__all__ += ['Q']\nclass Q:\n pass\n"}, + {"base/parts.py", "__all__ = ['Count']\nclass Count:\n pass\n"}, + {"gis/__init__.py", "from base import *\n"}, + {"gis/functions.py", "class Area:\n pass\n"}, + {"main.py", "from gis import functions, Q, Count\ndef run():\n functions.Area()\n" + " Q()\n Count()\n"}}; + ASSERT(python_expect("a submodule past that star", files, 5, "CALLS", "gis/functions.py", + "Area", "Class", 1)); + ASSERT(python_expect("a public name through that star", files, 5, "CALLS", "base/__init__.py", + "Q", "Class", 1)); + ASSERT(python_expect("a name that star itself got by a star", files, 5, "CALLS", + "base/parts.py", "Count", "Class", 1)); + } + { + /* A dynamic module stays open whatever its __all__ says. */ + const python_proof_file_t files[] = { + {"base/__init__.py", "__all__ = ['Q']\ndef __getattr__(name):\n return 1\n"}, + {"gis/__init__.py", "from base import *\n"}, + {"gis/functions.py", "class Area:\n pass\n"}, + {"main.py", "from gis import functions\ndef run():\n functions.Area()\n"}}; + ASSERT(python_expect("a star of a module with __getattr__", files, 4, "CALLS", + "gis/functions.py", "Area", "Class", 0)); + } + /* views/decorators: if/else twins of a nested def are one declaration. */ + { + const python_proof_file_t files[] = { + {"main.py", + "from functools import wraps\ndef helper():\n return 1\n" + "def deco(func, c):\n if c:\n async def inner(request):\n" + " helper()\n else:\n def inner(request):\n" + " helper()\n return wraps(func)(inner)\n"}}; + ASSERT(python_expect_from("calls from nested twin defs", files, 1, "main.py", "CALLS", + "main.py", "helper", "Function", 1)); + } + { + const python_proof_file_t files[] = { + {"main.py", + "def helper():\n return 1\ndef deco(c):\n def inner():\n helper()\n" + " def inner():\n return 2\n return inner\n"}}; + ASSERT(python_expect_from("two defs in a row are a rebinding", files, 1, "main.py", "CALLS", + "main.py", "helper", "Function", 0)); + } + PASS(); +} + SUITE(resolver_python) { RUN_TEST(resolver_python_ordinary_syntax_keeps_the_file); RUN_TEST(resolver_python_only_real_namespace_effects_are_dynamic); @@ -1530,7 +2798,36 @@ SUITE(resolver_python_members) { python_route_release(&python_routes[1]); } +SUITE(resolver_python_layout) { + RUN_TEST(resolver_python_stated_source_root_reaches_the_module); + RUN_TEST(resolver_python_namespace_package_under_a_source_root); + RUN_TEST(resolver_python_manifests_state_their_roots); + RUN_TEST(resolver_python_source_root_decoys_abstain); + RUN_TEST(resolver_python_workspace_members_see_each_other); + RUN_TEST(resolver_python_only_a_store_of_that_name_blocks_a_submodule); + RUN_TEST(resolver_python_writes_through_the_submodule_leave_its_identity); + RUN_TEST(resolver_python_module_getattr_only_answers_what_is_missing); + RUN_TEST(resolver_python_package_importing_its_own_child); + RUN_TEST(resolver_python_names_a_module_states_only_for_checkers); + RUN_TEST(resolver_python_overload_stubs_are_one_declaration); + RUN_TEST(resolver_python_loading_a_child_the_parent_imported); + python_route_release(&python_routes[0]); + python_route_release(&python_routes[1]); +} + SUITE(resolver_python_flow) { + RUN_TEST(resolver_python_special_methods_that_read_through_the_base); + RUN_TEST(resolver_python_returns_follow_the_receiver); + RUN_TEST(resolver_python_with_and_except_bind_what_they_say); + RUN_TEST(resolver_python_the_binder_every_path_passes_last); + RUN_TEST(resolver_python_annotated_containers_hold_their_element); + RUN_TEST(resolver_python_isinstance_narrows_where_it_holds); + python_route_release(&python_routes[0]); + python_route_release(&python_routes[1]); +} + +SUITE(resolver_python_shapes) { + RUN_TEST(resolver_python_class_and_module_shapes); RUN_TEST(resolver_python_rebound_receivers_abstain); RUN_TEST(resolver_python_module_level_instance_has_its_class); RUN_TEST(resolver_python_decorators_keep_identity_only_when_proven);