feat(resolver): recover Python edges a project's layout and ordinary writes had voided

CALLS recall against SCIP on six repositories rises from 80.7% to 95.5%, precision holding at 99.5%. A manifest now states source roots, so a src layout reaches its own package. A write blocks only the name it spells, not a whole package. Decorators keep the declaration unless the source proves the function dropped. Three false edges the baseline also had are fixed: a conditional star's first arm, t.x = make() reading the declared x, and a descriptor read as the instance.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-11-02 18:07:57 +01:00
1 parent d614236668
commit 8a0afb5fed
12 files changed
+4241 -694

No files matched your search

+2 -1
View File
@@ -612,7 +612,8 @@ PIPELINE_SRCS = \
src/pipeline/pass_cross_repo.c \
src/pipeline/artifact.c \
src/pipeline/pass_ensemble_routing.c \
src/pipeline/pass_pkgmap.c
src/pipeline/pass_pkgmap.c \
src/pipeline/py_layout.c
# SimHash / MinHash module
SIMHASH_SRCS = src/simhash/minhash.c
File diff suppressed because it is too large. Load diff
+36 -13
View File
@@ -80,14 +80,6 @@ typedef enum {
CBM_PY_MEMBER_OPAQUE /* conditionally defined: the name stops lookup */
} cbm_resolve_python_member_kind_t;
/* What a function does when used as a decorator, proven from its source. */
enum {
CBM_PY_SHAPE_DIRECT = 1u << 0,
CBM_PY_SHAPE_FACTORY = 1u << 1,
/* It hands the function to another decorator, named by delegate_chain. */
CBM_PY_SHAPE_DELEGATES = 1u << 2
};
typedef struct {
uint32_t owner_scope, chain_scope, at;
const char *owner, *name, *target;
@@ -95,9 +87,9 @@ typedef struct {
const char *const *decorators;
const char *return_chain, *value_chain, *type_chain;
bool none_value;
unsigned shape;
const char *delegate_chain;
bool delegate_applied;
bool discards; /* as a decorator: returns a constant or nothing on every path */
bool replaces; /* as a decorator: returns a value it never built from its parameter */
const char *spelled_decorator; /* `go = classmethod(go)`: the callee */
/* What the function returns, as access paths in its own scope. NULL when a
* return is not followable or the function is a generator. */
const char *const *returned;
@@ -129,6 +121,7 @@ typedef struct {
typedef struct {
const char *module_path;
uint32_t at;
bool annotation_only; /* under `if TYPE_CHECKING:` */
} cbm_resolve_python_star_t;
/* Build once per source file. Borrows result/source until the arena is freed.
@@ -286,8 +279,38 @@ typedef struct {
* non-module scopes; copy while the index is alive. */
/* Dynamic execution/module registry access remains an unknown project effect. */
bool cbm_resolve_binding_python_effects_complete(const cbm_resolve_binding_index_t *);
/* Conservatively includes writes and unknown alias/callee escapes. */
bool cbm_resolve_binding_python_import_escaped(const cbm_resolve_binding_index_t *, const char *);
/* A type annotation's chain as a value chain: `X` is X(), `X@seq` is X()@seq, `X@type` is X. */
const char *cbm_resolve_python_typed_text(CBMArena *arena, const char *type, const char *rest);
/* The class an isinstance test makes the longest narrowed prefix of this chain at this site.
* *consumed is how much of the chain it covers. NULL when no test applies. */
const char *cbm_resolve_binding_python_narrowed(const cbm_resolve_binding_index_t *index,
const char *chain, uint32_t scope, uint32_t site,
size_t *consumed);
/* A store through a proper prefix of this chain earlier in the scope holding the site: that
* prefix's length, and the chain it stored (NULL when not followable) with where to read it. */
bool cbm_resolve_binding_python_store_precedes(const cbm_resolve_binding_index_t *index,
const char *chain, uint32_t scope, uint32_t site,
size_t *prefix, const char **value,
uint32_t *value_at, uint32_t *value_scope);
/* Which binder of a local every path to this site passes last. VALUE names its value, NONE
* means the flow does not decide it, UNKNOWN that it does and its value cannot be followed. */
typedef enum {
CBM_PY_FLOW_NONE = 0,
CBM_PY_FLOW_VALUE,
CBM_PY_FLOW_UNKNOWN
} cbm_resolve_python_flow_t;
cbm_resolve_python_flow_t cbm_resolve_binding_python_reaching(
const cbm_resolve_binding_index_t *index, const char *name, size_t length, uint32_t scope,
uint32_t site, const char **chain, uint32_t *at, uint32_t *chain_scope, bool *typed);
/* The offset-th write this file makes through a module it bound to name. *member is the
* dotted path written below that module, NULL when any name can change. False past the
* last one. A read, a call and a hand-off to a callee are not writes. */
bool cbm_resolve_binding_python_import_write(const cbm_resolve_binding_index_t *, const char *name,
size_t offset, const char **member);
bool cbm_resolve_binding_python_names_come_at_run_time(const cbm_resolve_binding_index_t *index);
bool cbm_resolve_binding_python_namespace_closed(const cbm_resolve_binding_index_t *index);
size_t cbm_resolve_binding_python_namespace_size(const cbm_resolve_binding_index_t *index);
const char *cbm_resolve_binding_python_namespace_name(const cbm_resolve_binding_index_t *index,
+5 -4
View File
@@ -302,10 +302,11 @@ bool cbm_resolve_capabilities(CBMLanguage language, cbm_resolve_capabilities_t *
CBM_RESOLVE_CAP_DECLARED_TYPE | CBM_RESOLVE_CAP_MEMBER |
CBM_RESOLVE_CAP_TYPE_FLOW | CBM_RESOLVE_CAP_CALLABLE;
out->limitation =
"Imports, reexports and closed star imports; self, cls and super through C3 order; "
"receivers typed by construction or annotation; decorators proven to keep identity. "
"Untyped receivers, computed bases, class-body values a metaclass can replace and "
"unproven decorators abstain.";
"Imports, reexports and closed star imports, through source roots a manifest states; "
"self, cls and super through C3 order; receivers typed by construction, annotation, "
"isinstance, with or except; annotated containers; a decorator keeps the declaration "
"unless its source discards the function. Untyped receivers, computed bases, Enum "
"and NamedTuple members and classes reached by computed writes abstain.";
break;
case CBM_LANG_PHP:
out->adapter = "php_source";
+412 -127
View File
@@ -2,6 +2,7 @@
#include "foundation/hash_table.h"
#include "foundation/mem_core.h"
#include <stdlib.h>
#include <string.h>
/* Finite analysis per occurrence. Past either limit the receiver is unknown, the
@@ -9,6 +10,9 @@
* other would otherwise branch without end. */
enum { PY_MAX_DEPTH = 24, PY_MAX_MRO = 512, PY_MAX_REF_STEPS = 64, PY_MAX_EVAL_STEPS = 2048 };
/* Where a chain's root name ends, and each step after it. */
#define PY_CHAIN_STOPS ".([@"
typedef enum {
PY_REF_NONE = 0,
PY_REF_KEY, /* a shared solver fact */
@@ -65,28 +69,31 @@ typedef struct py_class py_class_t;
typedef struct {
py_ref_t result;
unsigned shape;
py_ref_t delegate;
bool delegate_applied;
bool discards; /* as a decorator: returns a constant or nothing */
bool replaces; /* as a bare decorator: returns a value not built from its parameter */
/* Unannotated: the class every return statement agrees on, settled at seal. */
py_ref_t *returned;
uint32_t returned_count;
const py_class_t *inferred;
bool returns_class; /* inferred is the class object itself, not an instance */
bool returns_receiver; /* every return is self, or cls(...): the receiver's class */
unsigned char inference; /* 0 pending, 1 in progress, 2 settled */
} py_function_t;
typedef struct {
const char *name, *target;
py_member_kind_t kind;
py_ref_t value; /* return type, property type, or class attribute value */
bool none_default; /* `x = None` in the class body: instances store the real value */
bool annotated; /* its value comes from an annotation */
py_ref_t value; /* return type, property type, or class attribute value */
bool none_default; /* `x = None` in the class body: instances store the real value */
bool spelled_decorator; /* `go = classmethod(go)`, with the builtin */
bool annotated; /* its value comes from an annotation */
} py_member_t;
typedef struct {
const char *name;
py_ref_t value;
bool none;
bool annotated; /* `self.x: T = ...` or `x: T` in the body */
} py_attribute_t;
struct py_class {
@@ -174,7 +181,7 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre
py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL};
if (!chain || depth > PY_MAX_DEPTH || ++*steps > PY_MAX_REF_STEPS)
return none;
size_t length = strcspn(chain, ".(");
size_t length = strcspn(chain, PY_CHAIN_STOPS);
const char *rest = chain + length;
/* super and type are the builtins only while nothing in scope binds the name. */
bool builtin = rest[0] == '(' &&
@@ -209,8 +216,19 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre
const char *open = strchr(chain, '(');
if (open && open[1] != ')' && open > chain) {
const char *spelled = cbm_arena_strndup(arena, chain, (size_t)(open - chain));
if (spelled && (!strcmp(spelled, "partial") || !strcmp(spelled, "functools.partial")) &&
cbm_resolve_binding_python_import_is(index, spelled, "functools.partial")) {
bool partial = spelled &&
(!strcmp(spelled, "partial") || !strcmp(spelled, "functools.partial")) &&
cbm_resolve_binding_python_import_is(index, spelled, "functools.partial");
/* list(x), sorted(x), reversed(x) hold what x holds, while they are the builtins. */
bool passing =
spelled &&
(!strcmp(spelled, "list") || !strcmp(spelled, "tuple") || !strcmp(spelled, "set") ||
!strcmp(spelled, "frozenset") || !strcmp(spelled, "sorted") ||
!strcmp(spelled, "reversed") || !strcmp(spelled, "iter") ||
!strcmp(spelled, "enumerate") || !strcmp(spelled, "zip")) &&
cbm_resolve_binding_python_root(index, spelled, strlen(spelled), scope, at).kind ==
CBM_PY_ROOT_UNBOUND;
if (partial || passing) {
int nesting = 0;
const char *close = open;
for (; *close; close++) {
@@ -218,10 +236,13 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre
if (!nesting)
break;
}
if (!*close || close[1])
if (!*close || (partial && close[1]))
return none;
const char *wrapped = cbm_arena_strndup(arena, open + 1, (size_t)(close - open - 1));
return py_ref_of_chain(index, arena, imports, wrapped, scope, at, depth + 1, steps);
const char *wrapped =
cbm_arena_sprintf(arena, "%.*s%s", (int)(close - open - 1), open + 1, close + 1);
return wrapped ? py_ref_of_chain(index, arena, imports, wrapped, scope, at, depth + 1,
steps)
: none;
}
}
cbm_resolve_python_root_t root =
@@ -244,7 +265,8 @@ static py_ref_t py_ref_of_chain(const cbm_resolve_binding_index_t *index, CBMAre
return none;
for (uint32_t i = 0; i < root.chain_count; i++) {
const char *arm = root.chains[i];
if (!strncmp(arm, chain, length) && (!arm[length] || strchr(".(", arm[length]))) {
if (!strncmp(arm, chain, length) &&
(!arm[length] || strchr(PY_CHAIN_STOPS, arm[length]))) {
tails[join->self_count++] = arm + length;
continue;
}
@@ -296,10 +318,10 @@ static py_ref_t py_ref_of_type(const cbm_resolve_binding_index_t *index, CBMAren
py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL};
if (!chain)
return none;
size_t length = strcspn(chain, ".(");
size_t length = strcspn(chain, PY_CHAIN_STOPS);
cbm_resolve_python_root_t root =
cbm_resolve_binding_python_root(index, chain, length, scope, 0);
const char *typed = cbm_arena_sprintf(arena, "%s()%s", chain, rest ? rest : "");
const char *typed = cbm_resolve_python_typed_text(arena, chain, rest ? rest : "");
if (!typed)
return none;
if (root.kind != CBM_PY_ROOT_ANNOTATION_IMPORT)
@@ -309,6 +331,82 @@ static py_ref_t py_ref_of_type(const cbm_resolve_binding_index_t *index, CBMAren
return key ? (py_ref_t){PY_REF_KEY, key, remainder, NULL} : none;
}
static py_ref_t py_ref_of_site(const cbm_resolve_binding_index_t *index, CBMArena *arena,
const cbm_py_import_lookup_t *imports, const char *chain,
uint32_t scope, uint32_t site, unsigned depth, unsigned *steps);
/* The chain reduced through the binder every path to the site passes last. The value was read
* where that binder stands, so `x = x.clone()` follows x to the binder before it. */
static py_ref_t py_ref_of_reach(const cbm_resolve_binding_index_t *index, CBMArena *arena,
const cbm_py_import_lookup_t *imports, const char *chain,
uint32_t scope, uint32_t site, unsigned depth, unsigned *steps) {
py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL};
size_t length = strcspn(chain, PY_CHAIN_STOPS);
const char *rest = chain + length, *value = NULL;
uint32_t at = 0, chain_scope = scope;
bool typed = false;
cbm_resolve_python_flow_t flow = cbm_resolve_binding_python_reaching(
index, chain, length, scope, site, &value, &at, &chain_scope, &typed);
if (flow == CBM_PY_FLOW_NONE)
return py_ref_of_chain(index, arena, imports, chain, scope, 0, depth, steps);
if (flow == CBM_PY_FLOW_UNKNOWN || depth > PY_MAX_DEPTH || ++*steps > PY_MAX_REF_STEPS)
return none;
if (typed)
return py_ref_of_type(index, arena, imports, value, rest, chain_scope, depth + 1, steps);
const char *joined = cbm_arena_sprintf(arena, "%s%s", value, rest);
return joined ? py_ref_of_site(index, arena, imports, joined, chain_scope, at, depth + 1, steps)
: none;
}
/* What a store through a prefix of the chain leaves there, held against what that prefix is
* declared to hold. Reading on is sure only where the two agree. */
static py_ref_t py_ref_past_store(const cbm_resolve_binding_index_t *index, CBMArena *arena,
const cbm_py_import_lookup_t *imports, const char *chain,
size_t prefix, const char *value, uint32_t value_at,
uint32_t value_scope, uint32_t scope, uint32_t site,
unsigned depth, unsigned *steps) {
py_ref_t none = {PY_REF_NONE, NULL, NULL, NULL};
if (!value || depth > PY_MAX_DEPTH || ++*steps > PY_MAX_REF_STEPS)
return none;
py_join_t *join = cbm_arena_calloc(arena, sizeof(*join));
py_ref_t *arms = cbm_arena_calloc(arena, 2 * sizeof(*arms));
const char **tails = cbm_arena_calloc(arena, sizeof(*tails));
const char *declared = cbm_arena_strndup(arena, chain, prefix);
if (!join || !arms || !tails || !declared)
return none;
arms[0] = py_ref_of_reach(index, arena, imports, declared, scope, site, depth + 1, steps);
arms[1] = py_ref_of_site(index, arena, imports, value, value_scope, value_at, depth + 1, steps);
if (arms[0].kind == PY_REF_NONE || arms[1].kind == PY_REF_NONE)
return none;
join->arms = arms;
join->arm_count = 2;
join->self_tails = tails;
return (py_ref_t){.kind = PY_REF_JOIN, .join = join, .tail = chain + prefix};
}
/* The chain at this site: what a narrowing says it is, what a store left there, or its binder. */
static py_ref_t py_ref_of_site(const cbm_resolve_binding_index_t *index, CBMArena *arena,
const cbm_py_import_lookup_t *imports, const char *chain,
uint32_t scope, uint32_t site, unsigned depth, unsigned *steps) {
/* `if isinstance(x, C):` says what x is here, while isinstance is the builtin. */
size_t consumed = 0;
const char *narrowed =
cbm_resolve_binding_python_narrowed(index, chain, scope, site, &consumed);
if (narrowed &&
cbm_resolve_binding_python_root(index, "isinstance", strlen("isinstance"), scope, 0).kind ==
CBM_PY_ROOT_UNBOUND)
return py_ref_of_type(index, arena, imports, narrowed, chain + consumed, scope, depth + 1,
steps);
size_t prefix = 0;
const char *stored = NULL;
uint32_t stored_at = 0, stored_scope = scope;
if (cbm_resolve_binding_python_store_precedes(index, chain, scope, site, &prefix, &stored,
&stored_at, &stored_scope))
return py_ref_past_store(index, arena, imports, chain, prefix, stored, stored_at,
stored_scope, scope, site, depth, steps);
return py_ref_of_reach(index, arena, imports, chain, scope, site, depth, steps);
}
static py_ref_t py_keep_ref(cbm_py_members_t *members, py_ref_t ref) {
ref.name = py_keep(members, ref.name);
ref.tail = py_keep(members, ref.tail ? ref.tail : "");
@@ -400,8 +498,9 @@ static py_class_t *py_class_for_scope(py_class_t **by_scope, size_t count, uint3
return scope < count ? by_scope[scope] : NULL;
}
/* Sort a class's members and collapse duplicates: a getter with its setter stays
* one property; any other repetition is not one provable declaration. */
/* Sort a class's members and collapse duplicates: a getter with its setter stays one property,
* a def with `go = classmethod(go)` stays that def; any other repetition is not one provable
* declaration. */
static void py_finish_members(py_class_t *entry) {
if (entry->member_count > 1)
qsort(entry->members, entry->member_count, sizeof(*entry->members), py_compare_member);
@@ -410,8 +509,13 @@ static void py_finish_members(py_class_t *entry) {
py_member_t *member = &entry->members[i];
if (used && !strcmp(entry->members[used - 1].name, member->name)) {
py_member_t *prior = &entry->members[used - 1];
if (prior->kind != PY_MEMBER_PROPERTY || member->kind != PY_MEMBER_PROPERTY)
prior->kind = PY_MEMBER_OPAQUE;
if (prior->spelled_decorator && member->kind == PY_MEMBER_FUNCTION)
*prior = *member;
else if (!member->spelled_decorator || prior->kind != PY_MEMBER_FUNCTION)
prior->kind =
prior->kind == PY_MEMBER_PROPERTY && member->kind == PY_MEMBER_PROPERTY
? PY_MEMBER_PROPERTY
: PY_MEMBER_OPAQUE;
continue;
}
entry->members[used++] = *member;
@@ -438,10 +542,8 @@ static bool py_note_function(cbm_py_members_t *members, const cbm_resolve_bindin
return false;
function->result = py_capture_type(members, index, scratch, imports, member->return_chain,
member->chain_scope);
function->shape = member->shape;
function->delegate_applied = member->delegate_applied;
function->delegate =
py_capture(members, index, scratch, imports, member->delegate_chain, member->chain_scope);
function->discards = member->discards;
function->replaces = member->replaces;
uint32_t count = 0;
while (member->returned && member->returned[count])
count++;
@@ -549,6 +651,7 @@ bool cbm_py_members_add_file(cbm_py_members_t *members, const cbm_resolve_bindin
py_attribute_t *attribute = &entry->attributes[entry->attribute_count++];
attribute->name = py_keep(members, member->name);
attribute->none = member->none_value;
attribute->annotated = member->type_chain != NULL;
attribute->value = member->type_chain
? py_capture_type(members, index, &scratch, imports,
member->type_chain, member->chain_scope)
@@ -561,6 +664,11 @@ bool cbm_py_members_add_file(cbm_py_members_t *members, const cbm_resolve_bindin
slot->target = py_keep(members, member->target);
slot->kind = py_member_kind(index, member);
slot->none_default = member->kind == CBM_PY_MEMBER_VALUE && member->none_value;
slot->spelled_decorator = member->spelled_decorator &&
cbm_resolve_binding_python_root(index, member->spelled_decorator,
strlen(member->spelled_decorator),
member->owner_scope, 0)
.kind == CBM_PY_ROOT_UNBOUND;
slot->annotated = member->type_chain != NULL;
slot->value = member->kind == CBM_PY_MEMBER_FUNCTION
? py_capture_type(members, index, &scratch, imports, member->return_chain,
@@ -574,7 +682,10 @@ bool cbm_py_members_add_file(cbm_py_members_t *members, const cbm_resolve_bindin
ok = false;
break;
}
if (member->kind == CBM_PY_MEMBER_FUNCTION)
/* A def under a condition has no provable identity, and its returns still say whether
* a special method defers to the base. */
if (member->kind == CBM_PY_MEMBER_FUNCTION ||
(member->kind == CBM_PY_MEMBER_OPAQUE && member->returned))
ok = py_note_function(members, index, &scratch, imports, member);
}
for (size_t c = 0; ok && c < classes; c++) {
@@ -656,7 +767,9 @@ typedef enum {
PY_VALUE_INSTANCE,
PY_VALUE_SUPER,
PY_VALUE_FUNCTION,
PY_VALUE_EXTERNAL
PY_VALUE_EXTERNAL,
PY_VALUE_SEQ, /* holds and yields instances of cls */
PY_VALUE_MAP /* holds instances of cls by key; what it yields is the keys */
} py_value_kind_t;
typedef struct {
@@ -666,6 +779,9 @@ typedef struct {
const py_ref_t *result; /* annotated return of a function */
const char *declaration; /* the member read last, for a reference to it */
const char *owner;
const py_class_t *receiver; /* the class a method was read through */
/* An instance, or class, standing for whatever self or cls is at run time. */
bool of_receiver;
} py_value_t;
typedef struct {
@@ -747,14 +863,46 @@ static const py_member_t *py_lookup(const py_class_t *cls, const char *name, uin
return NULL;
}
/* What instances store under this name, across the certain linearization. All
/* What instances store under this name: the nearest class in the certain linearization that
* stores it, up to the class that declares the member when one does. A store further along is
* another declaration of the name, and the nearer one selects, as it does for a method. A class's
* typed stores must agree; a None store says nothing about the class. */
static bool py_instance_attribute(const py_eval_t *eval, const py_class_t *cls, const char *name,
py_value_t *out, unsigned depth) {
const py_class_t *declarer, py_value_t *out, unsigned depth) {
bool present = false, unknown = false;
py_value_t agreed = {.kind = PY_VALUE_UNKNOWN};
/* An annotation is the attribute's declared type wherever it stands, and what a subclass
* stores without one is held to it. */
const py_attribute_t *declared = NULL;
for (uint32_t i = 0; i < cls->reliable && !declared; i++) {
const py_class_t *owner = cls->mro[i];
if (i && declarer && cls->mro[i - 1] == declarer)
break;
py_attribute_t key = {.name = name};
const py_attribute_t *found =
owner->attribute_count ? bsearch(&key, owner->attributes, owner->attribute_count,
sizeof(*owner->attributes), py_compare_attribute)
: NULL;
if (!found)
continue;
while (found > owner->attributes && !strcmp(found[-1].name, name))
found--;
for (; found < owner->attributes + owner->attribute_count && !strcmp(found->name, name);
found++)
if (found->annotated) {
declared = found;
break;
}
}
if (declared) {
*out = py_eval_ref(eval, &declared->value, depth + 1);
return true;
}
for (uint32_t i = 0; i < cls->reliable && !unknown; i++) {
const py_class_t *owner = cls->mro[i];
if (i && declarer && cls->mro[i - 1] == declarer)
break;
py_attribute_t key = {.name = name};
const py_attribute_t *found =
owner->attribute_count ? bsearch(&key, owner->attributes, owner->attribute_count,
@@ -770,46 +918,60 @@ static bool py_instance_attribute(const py_eval_t *eval, const py_class_t *cls,
if (found->none)
continue;
py_value_t value = py_eval_ref(eval, &found->value, depth + 1);
if (value.kind != PY_VALUE_INSTANCE ||
(agreed.kind != PY_VALUE_UNKNOWN && agreed.cls != value.cls)) {
bool holds = value.kind == PY_VALUE_INSTANCE || value.kind == PY_VALUE_SEQ ||
value.kind == PY_VALUE_MAP;
if (!holds || (agreed.kind != PY_VALUE_UNKNOWN &&
(agreed.cls != value.cls || agreed.kind != value.kind))) {
unknown = true;
break;
}
agreed = value;
}
if (agreed.kind != PY_VALUE_UNKNOWN)
break;
}
*out = unknown ? (py_value_t){.kind = PY_VALUE_UNKNOWN} : agreed;
return present;
}
/* A metaclass or __init_subclass__ can replace what a class body assigned: a Django field
* reads as its column value, an Enum member as the enum. A base we cannot resolve at all
* could be anything. An external class is taken to keep it, as unittest.TestCase keeps
* rf = RequestFactory(); the known exceptions from the stdlib count as a metaclass. */
/* What a class body assigned is the attribute's declared value, and a read selects that:
* what a metaclass does to it is not a static fact, and a method called on such a read has to
* exist on the declared class to resolve at all. Enum and NamedTuple are the exceptions the
* standard library states: a member reads as the enum. A base we cannot resolve at all could
* be anything. */
static bool py_class_body_rewritable(const py_class_t *cls) {
if (cls->uncertain)
return true;
for (uint32_t i = 0; i < cls->mro_count; i++) {
const py_class_t *entry = cls->mro[i];
if (entry->metaclass || py_own_member(entry, "__init_subclass__"))
for (uint32_t i = 0; i < cls->mro_count; i++)
if (cls->mro[i]->metaclass)
return true;
}
return false;
}
static py_value_t py_function_returns(const py_eval_t *eval, const char *qualified_name,
const py_class_t *receiver, bool of_receiver, unsigned depth);
static py_value_t py_special_step(const py_eval_t *eval, const py_value_t *value, const char *name,
unsigned depth);
static py_value_t py_read_member(const py_eval_t *eval, const py_value_t *value, const char *name,
unsigned depth) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
const py_class_t *cls = value->cls;
if (value->kind == PY_VALUE_INSTANCE && !strcmp(name, "__class__"))
return (py_value_t){.kind = PY_VALUE_CLASS, .cls = cls};
return (py_value_t){.kind = PY_VALUE_CLASS, .cls = cls, .of_receiver = value->of_receiver};
if (value->kind == PY_VALUE_CLASS && !strcmp(name, "<super>"))
return (py_value_t){.kind = PY_VALUE_SUPER, .cls = cls};
const py_class_t *owner = NULL;
const py_member_t *member = py_lookup(cls, name, value->kind == PY_VALUE_SUPER ? 1 : 0, &owner);
if (value->kind == PY_VALUE_INSTANCE) {
/* `cls.user = User()` in a classmethod is read back through the class, and only code that
* stored it that way can read it so. What the class declares is what its class answers:
* `self.go = make()` says nothing about `Child.go`. */
if (value->kind == PY_VALUE_INSTANCE ||
(value->kind == PY_VALUE_CLASS && (!member || member->none_default))) {
py_value_t stored;
if (py_instance_attribute(eval, cls, name, &stored, depth)) {
if (py_instance_attribute(eval, cls, name, member ? owner : NULL, &stored, depth)) {
if (member && member->kind != PY_MEMBER_PROPERTY && !member->none_default)
return unknown;
if (!member || member->none_default)
@@ -828,17 +990,36 @@ static py_value_t py_read_member(const py_eval_t *eval, const py_value_t *value,
.qualified_name = member->target,
.result = &member->value,
.declaration = member->target,
.owner = owner->qualified_name};
.owner = owner->qualified_name,
.receiver = cls,
.of_receiver = value->of_receiver};
case PY_MEMBER_PROPERTY: {
py_value_t read = value->kind == PY_VALUE_INSTANCE || value->kind == PY_VALUE_SUPER
? py_eval_ref(eval, &member->value, depth + 1)
: unknown;
bool instance = value->kind == PY_VALUE_INSTANCE || value->kind == PY_VALUE_SUPER;
py_value_t read =
!instance ? unknown
: member->value.kind != PY_REF_NONE
? py_eval_ref(eval, &member->value, depth + 1)
: py_function_returns(eval, member->target, cls, value->of_receiver, depth + 1);
read.declaration = member->target;
read.owner = owner->qualified_name;
return read;
}
case PY_MEMBER_VALUE: {
/* A class declared in this class's body. */
if (member->value.kind == PY_REF_NONE) {
const py_class_t *nested =
py_class(eval->members,
cbm_arena_sprintf(eval->arena, "%s.%s", owner->qualified_name, name));
if (nested)
return (py_value_t){.kind = PY_VALUE_CLASS, .cls = nested};
}
py_value_t read = py_eval_ref(eval, &member->value, depth + 1);
/* A class-body value whose class has __get__ is a descriptor: a read runs that. */
const py_class_t *getter = NULL;
if (read.kind == PY_VALUE_INSTANCE && !member->annotated &&
py_lookup(read.cls, "__get__", 0, &getter))
return py_special_step(eval, &read, "__get__", depth + 1);
/* An annotation states what a read yields, whatever the machinery does. */
return read.kind == PY_VALUE_INSTANCE && !member->annotated && py_class_body_rewritable(cls)
? unknown
@@ -858,32 +1039,55 @@ static py_value_t py_read_member(const py_eval_t *eval, const py_value_t *value,
static void py_infer_returns(const py_eval_t *eval, py_function_t *function, unsigned depth) {
function->inference = 1;
const py_class_t *agreed = NULL;
bool known = function->returned_count > 0 && depth <= PY_MAX_DEPTH;
bool known = function->returned_count > 0 && depth <= PY_MAX_DEPTH, receiver = known;
py_value_kind_t kind = PY_VALUE_UNKNOWN;
for (uint32_t i = 0; known && i < function->returned_count; i++) {
py_value_t value = py_eval_ref(eval, &function->returned[i], depth + 1);
known = value.kind == PY_VALUE_INSTANCE && (!agreed || agreed == value.cls);
known = (value.kind == PY_VALUE_INSTANCE || value.kind == PY_VALUE_CLASS) &&
(!agreed || (agreed == value.cls && kind == value.kind));
agreed = value.cls;
kind = value.kind;
receiver = receiver && value.of_receiver;
}
function->inferred = known ? agreed : NULL;
function->returns_class = known && kind == PY_VALUE_CLASS;
function->returns_receiver = known && receiver;
function->inference = 2;
}
/* What an unannotated function returns. `return self` in an inherited method is the class it
* was read through, as a type checker reads an unannotated self: chord(...).set() is a chord
* although set is declared on Signature. */
static py_value_t py_function_returns(const py_eval_t *eval, const char *qualified_name,
const py_class_t *receiver, bool of_receiver,
unsigned depth) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
py_function_t *function = cbm_ht_get(eval->members->functions, qualified_name);
if (function && function->inference == 0)
py_infer_returns(eval, function, depth);
if (!function || !function->inferred)
return unknown;
return (py_value_t){.kind = function->returns_class ? PY_VALUE_CLASS : PY_VALUE_INSTANCE,
.cls =
function->returns_receiver && receiver ? receiver : function->inferred,
.of_receiver = function->returns_receiver && of_receiver};
}
static py_value_t py_call(const py_eval_t *eval, const py_value_t *value, unsigned depth) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
switch (value->kind) {
case PY_VALUE_CLASS:
/* __new__ can return anything. */
return value->cls->custom_new ? unknown
: (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = value->cls};
: (py_value_t){.kind = PY_VALUE_INSTANCE,
.cls = value->cls,
.of_receiver = value->of_receiver};
case PY_VALUE_FUNCTION: {
if (value->result && value->result->kind != PY_REF_NONE)
return py_eval_ref(eval, value->result, depth + 1);
py_function_t *function = cbm_ht_get(eval->members->functions, value->qualified_name);
if (function && function->inference == 0)
py_infer_returns(eval, function, depth);
return function && function->inferred
? (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = function->inferred}
: unknown;
return py_function_returns(eval, value->qualified_name, value->receiver, value->of_receiver,
depth);
}
case PY_VALUE_INSTANCE: {
const py_class_t *owner = NULL;
@@ -896,6 +1100,79 @@ static py_value_t py_call(const py_eval_t *eval, const py_value_t *value, unsign
}
}
/* A special method of a project class answering a container step. */
static py_value_t py_special_step(const py_eval_t *eval, const py_value_t *value, const char *name,
unsigned depth) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
if (value->kind != PY_VALUE_INSTANCE)
return unknown;
const py_class_t *owner = NULL;
const py_member_t *member = py_lookup(value->cls, name, 0, &owner);
if (!member || member->kind != PY_MEMBER_FUNCTION)
return unknown;
py_value_t bound = {.kind = PY_VALUE_FUNCTION,
.qualified_name = member->target,
.result = &member->value,
.receiver = value->cls,
.of_receiver = value->of_receiver};
return py_call(eval, &bound, depth);
}
/* x[k], x[a:b], and iterating or wrapping x, as the annotation said. */
static py_value_t py_container_step(const py_eval_t *eval, const py_value_t *value,
const char *step, size_t length, unsigned depth) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
if (value->kind == PY_VALUE_EXTERNAL)
return *value;
bool seq = value->kind == PY_VALUE_SEQ, map = value->kind == PY_VALUE_MAP;
py_value_t element = {.kind = PY_VALUE_INSTANCE, .cls = value->cls};
if (length == 4 && !strncmp(step, "@seq", 4)) {
return value->kind == PY_VALUE_INSTANCE
? (py_value_t){.kind = PY_VALUE_SEQ, .cls = value->cls}
: unknown;
}
if (length == 4 && !strncmp(step, "@map", 4)) {
return value->kind == PY_VALUE_INSTANCE
? (py_value_t){.kind = PY_VALUE_MAP, .cls = value->cls}
: unknown;
}
/* d.items() against an annotated map: a class's own items() means what it says. */
if (length == 7 && !strncmp(step, "@values", 7))
return map ? (py_value_t){.kind = PY_VALUE_SEQ, .cls = value->cls} : unknown;
if (length == 5 && !strncmp(step, "@iter", 5)) {
if (seq)
return element;
if (map)
return unknown;
py_value_t iterator = py_special_step(eval, value, "__iter__", depth);
return iterator.kind == PY_VALUE_SEQ
? (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = iterator.cls}
: unknown;
}
if (length == 2 && !strncmp(step, "[]", 2)) {
if (seq || map)
return element;
return py_special_step(eval, value, "__getitem__", depth);
}
if (length == 3 && !strncmp(step, "[:]", 3))
return seq ? *value : py_special_step(eval, value, "__getitem__", depth);
return unknown;
}
/* The container methods that hand back what it holds. */
static py_value_t py_container_method(const py_value_t *value, const char *name) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
py_value_t element = {.kind = PY_VALUE_INSTANCE, .cls = value->cls};
if (!strcmp(name, "copy"))
return *value;
if (value->kind == PY_VALUE_SEQ)
return !strcmp(name, "pop") ? element : unknown;
if (!strcmp(name, "get") || !strcmp(name, "pop") || !strcmp(name, "setdefault"))
return element;
return !strcmp(name, "values") ? (py_value_t){.kind = PY_VALUE_SEQ, .cls = value->cls}
: unknown;
}
static py_value_t py_apply_tail(const py_eval_t *eval, py_value_t value, const char *tail,
unsigned depth) {
py_value_t unknown = {.kind = PY_VALUE_UNKNOWN};
@@ -905,14 +1182,33 @@ static py_value_t py_apply_tail(const py_eval_t *eval, py_value_t value, const c
tail += 2;
continue;
}
if (tail[0] == '@' || tail[0] == '[') {
size_t step =
tail[0] == '[' ? strcspn(tail, "]") + 1 : 1 + strcspn(tail + 1, PY_CHAIN_STOPS);
value = py_container_step(eval, &value, tail, step, depth);
tail += step;
continue;
}
if (tail[0] != '.')
return unknown;
tail++;
size_t length = strcspn(tail, ".(");
size_t length = strcspn(tail, PY_CHAIN_STOPS);
const char *name = cbm_arena_strndup(eval->arena, tail, length);
if (!name)
return unknown;
tail += length;
/* A with statement on what a contextmanager generator yields. */
if (value.kind == PY_VALUE_SEQ &&
(!strcmp(name, "__enter__") || !strcmp(name, "__aenter__")) && tail[0] == '(') {
value = (py_value_t){.kind = PY_VALUE_INSTANCE, .cls = value.cls};
tail += 2;
continue;
}
if ((value.kind == PY_VALUE_SEQ || value.kind == PY_VALUE_MAP) && tail[0] == '(') {
value = py_container_method(&value, name);
tail += 2;
continue;
}
switch (value.kind) {
case PY_VALUE_MODULE: {
const char *target = NULL;
@@ -964,7 +1260,8 @@ static py_value_t py_eval_ref(const py_eval_t *eval, const py_ref_t *ref, unsign
value = (py_value_t){.kind = ref->kind == PY_REF_SELF ? PY_VALUE_INSTANCE
: ref->kind == PY_REF_CLASS ? PY_VALUE_CLASS
: PY_VALUE_SUPER,
.cls = cls};
.cls = cls,
.of_receiver = true};
break;
}
case PY_REF_JOIN: {
@@ -994,75 +1291,36 @@ static py_value_t py_eval_ref(const py_eval_t *eval, const py_ref_t *ref, unsign
typedef enum { PY_KEEP_NONE = 0, PY_KEEP_IDENTITY, PY_KEEP_PROPERTY } py_keep_t;
/* Standard library decorators whose result is the function or class itself, or a
* thin wrapper that calls it. Facts about those modules, not about this project. */
static bool py_stdlib_keeps_identity(const char *canonical, bool applied) {
static const char *const direct[] = {"abc.abstractmethod",
"typing.final",
"typing.override",
"typing.no_type_check",
"typing_extensions.final",
"typing_extensions.override",
"functools.lru_cache",
"functools.cache",
"functools.total_ordering",
"contextlib.contextmanager",
"contextlib.asynccontextmanager",
"dataclasses.dataclass",
"enum.unique",
"typing.runtime_checkable",
"atexit.register",
"typing_extensions.runtime_checkable",
NULL};
static const char *const called[] = {"functools.lru_cache", "functools.wraps",
"dataclasses.dataclass", NULL};
for (const char *const *name = applied ? called : direct; *name; name++)
if (!strcmp(*name, canonical))
return true;
return false;
}
static bool py_stdlib_property(const char *canonical) {
return !strcmp(canonical, "functools.cached_property") ||
!strcmp(canonical, "abc.abstractproperty");
}
/* The source proves it keeps identity in this form, or hands the function on to a
* decorator that does. */
static bool py_function_keeps(const py_eval_t *eval, const char *qualified_name, bool applied,
unsigned depth) {
const py_function_t *function = cbm_ht_get(eval->members->functions, qualified_name);
if (!function || depth > 4)
return false;
if (function->shape & (applied ? CBM_PY_SHAPE_FACTORY : CBM_PY_SHAPE_DIRECT))
return true;
if (applied || !(function->shape & CBM_PY_SHAPE_DELEGATES))
return false;
py_value_t next = py_eval_ref(eval, &function->delegate, depth + 1);
return next.kind == PY_VALUE_FUNCTION &&
py_function_keeps(eval, next.qualified_name, function->delegate_applied, depth + 1);
/* functools.cached_property, kombu's, werkzeug's, abc.abstractproperty: reading such a member
* runs the function, and what a site then calls is its value. Saying so only ever withholds
* a call. */
static bool py_external_property(const char *canonical) {
const char *dot = strrchr(canonical, '.');
const char *name = dot ? dot + 1 : canonical;
size_t length = strlen(name);
return length >= 8 && !strcmp(name + length - 8, "property");
}
/* A decorator does not change which declaration a name selects. What runs when the name is
* called can differ, as it can behind any dispatch, and CALLS is not that. So every decorator
* keeps the declaration, the way a type checker reads one it cannot see through, unless the
* source proves it discards the function. */
static py_keep_t py_decorator_verdict(const py_eval_t *eval, const py_decorator_t *decorator) {
py_ref_t bare = decorator->ref;
if (bare.kind == PY_REF_UNBOUND && (!bare.tail || !bare.tail[0])) {
if (decorator->applied)
return PY_KEEP_NONE;
if (!strcmp(bare.name, "staticmethod") || !strcmp(bare.name, "classmethod"))
return PY_KEEP_IDENTITY;
return !strcmp(bare.name, "property") ? PY_KEEP_PROPERTY : PY_KEEP_NONE;
}
if (bare.kind == PY_REF_UNBOUND && (!bare.tail || !bare.tail[0]))
return !decorator->applied && !strcmp(bare.name, "property") ? PY_KEEP_PROPERTY
: PY_KEEP_IDENTITY;
py_value_t value = py_eval_ref(eval, &bare, 0);
if (value.kind == PY_VALUE_EXTERNAL && decorator->canonical) {
if (py_stdlib_keeps_identity(decorator->canonical, decorator->applied))
return PY_KEEP_IDENTITY;
return !decorator->applied && py_stdlib_property(decorator->canonical) ? PY_KEEP_PROPERTY
: PY_KEEP_NONE;
if (value.kind == PY_VALUE_EXTERNAL && decorator->canonical)
return !decorator->applied && py_external_property(decorator->canonical) ? PY_KEEP_PROPERTY
: PY_KEEP_IDENTITY;
if (value.kind == PY_VALUE_FUNCTION) {
const py_function_t *function = cbm_ht_get(eval->members->functions, value.qualified_name);
return function && (function->discards || (function->replaces && !decorator->applied))
? PY_KEEP_NONE
: PY_KEEP_IDENTITY;
}
if (value.kind == PY_VALUE_FUNCTION)
return py_function_keeps(eval, value.qualified_name, decorator->applied, 0)
? PY_KEEP_IDENTITY
: PY_KEEP_NONE;
/* A classmethod or staticmethod subclass binds the same function. */
for (uint32_t i = 0;
value.kind == PY_VALUE_CLASS && !decorator->applied && i < value.cls->base_count; i++) {
@@ -1074,7 +1332,7 @@ static py_keep_t py_decorator_verdict(const py_eval_t *eval, const py_decorator_
/* A descriptor class: `@cached_property` reads run the function once. */
if (value.kind == PY_VALUE_CLASS && !decorator->applied && py_own_member(value.cls, "__get__"))
return PY_KEEP_PROPERTY;
return PY_KEEP_NONE;
return PY_KEEP_IDENTITY;
}
bool cbm_py_members_decide_decorated(cbm_py_members_t *members, const cbm_py_members_env_t *env,
@@ -1306,11 +1564,36 @@ static uint32_t py_single_chain(const py_class_t *cls, const py_class_t **mro, u
return reliable;
}
/* Base[int] asks the class, and a __class_getitem__ can answer with anything. */
static bool py_answers_subscript(const py_class_t *cls) {
for (uint32_t i = 0; i < cls->mro_count; i++)
if (py_own_member(cls->mro[i], "__class_getitem__"))
/* This special method answers through the base on some path: `return super().__new__(cls)`,
* or django's __getattribute__ reading `super().__getattribute__(name)`. Then the declared
* members are still what a read finds, and a call of the class still makes an instance. */
static bool py_defers_to_base(const cbm_py_members_t *members, const py_member_t *member,
const char *tail) {
const py_function_t *function = cbm_ht_get(members->functions, member->target);
size_t tail_length = strlen(tail);
for (uint32_t i = 0; function && i < function->returned_count; i++) {
const py_ref_t *returned = &function->returned[i];
size_t length = returned->tail ? strlen(returned->tail) : 0;
bool base = returned->kind == PY_REF_SUPER ||
(returned->kind == PY_REF_UNBOUND && !strcmp(returned->name, "object")) ||
(length > tail_length &&
!strncmp(returned->tail + length - tail_length - strlen(".<super>"),
".<super>", strlen(".<super>")));
if (base && length >= tail_length && !strcmp(returned->tail + length - tail_length, tail))
return true;
}
return false;
}
/* Base[int] asks the class, and a __class_getitem__ can answer with anything. click's fills
* in a parameter and answers through super, so Base[int] is still Base. */
static bool py_answers_subscript(const cbm_py_members_t *members, const py_class_t *cls) {
for (uint32_t i = 0; i < cls->mro_count; i++) {
const py_member_t *member = py_own_member(cls->mro[i], "__class_getitem__");
if (member && !py_defers_to_base(members, member, ".__class_getitem__()"))
return true;
}
return false;
}
@@ -1346,7 +1629,7 @@ static void py_seal_class(const py_eval_t *eval, cbm_py_members_t *members, py_c
}
py_seal_class(eval, members, (py_class_t *)base, depth + 1);
if (!base->sealed ||
(b < 64 && ((cls->subscripted >> b) & 1) && py_answers_subscript(base))) {
(b < 64 && ((cls->subscripted >> b) & 1) && py_answers_subscript(members, base))) {
unsealed = true;
continue;
}
@@ -1382,8 +1665,10 @@ static void py_seal_class(const py_eval_t *eval, cbm_py_members_t *members, py_c
reliable = py_single_chain(cls, mro, mro_count);
cls->reliable = reliable;
for (uint32_t i = 0; i < reliable; i++) {
cls->intercepts |= py_own_member(mro[i], "__getattribute__") != NULL;
cls->custom_new |= py_own_member(mro[i], "__new__") != NULL;
const py_member_t *read = py_own_member(mro[i], "__getattribute__");
const py_member_t *made = py_own_member(mro[i], "__new__");
cls->intercepts |= read && !py_defers_to_base(members, read, ".__getattribute__()");
cls->custom_new |= made && !py_defers_to_base(members, made, ".__new__()");
}
cls->visiting = false;
cls->sealed = true;
@@ -1495,7 +1780,7 @@ bool cbm_py_members_resolve(const cbm_py_members_t *members, const cbm_py_member
!cbm_resolve_binding_python_site(index, name, start, end, origin, &chain, &scope))
return false;
unsigned ref_steps = 0, eval_steps = 0;
py_ref_t ref = py_ref_of_chain(index, arena, imports, chain, scope, 0, 0, &ref_steps);
py_ref_t ref = py_ref_of_site(index, arena, imports, chain, scope, start, 0, &ref_steps);
if (ref.kind == PY_REF_NONE || ref.kind == PY_REF_UNBOUND)
return false;
py_eval_t eval = {.members = members, .env = env, .arena = arena, .steps = &eval_steps};
+340 -25
View File
@@ -15,6 +15,8 @@
#include "pipeline/pass_lsp_cross.h"
#include "pipeline/import_resolve.h"
#include "pipeline/resolver_ecma.h"
#include "pipeline/py_layout.h"
#include "resolve/capabilities.h"
#include "resolve/policy.h"
#include "resolve/rust_names.h"
@@ -1251,6 +1253,77 @@ static void parse_package_swift(const char *source, int source_len, const char *
swift_scan_membership_only(source, end, rel_path, ".executableTarget(", "Sources", entries);
}
/* Python: the source roots a manifest states, one repository directory per line. The
* conventional `src` beside a manifest is a root for every build backend, as a flat
* layout is its own directory. A root that holds no such module answers nothing. */
typedef struct {
const char *directory;
char roots[PKGMAP_PATH_BUF];
size_t used;
cbm_pkg_entries_t *entries;
} python_roots_t;
static void python_add_root(void *context, const char *root, size_t length) {
python_roots_t *roots = context;
char written[PKGMAP_PATH_BUF], path[PKGMAP_PATH_BUF];
if (length >= sizeof(written))
return;
memcpy(written, root, length);
written[length] = '\0';
if (strpbrk(written, "*?\\\n") || written[0] == '/' ||
!cbm_repo_rel_join(roots->directory, written, path, sizeof(path)) || !path[0])
return;
size_t size = strlen(path);
if (roots->used + size + 2 > sizeof(roots->roots))
return;
/* Only a root apart from the manifest makes everything below it a module: a file
* in a flat project's own directory can still be a script. */
if (strcmp(path, roots->directory) != 0)
node_collect(roots->entries, "py-root", path, NULL, "1");
memcpy(roots->roots + roots->used, path, size);
roots->used += size;
roots->roots[roots->used++] = '\n';
roots->roots[roots->used] = '\0';
}
typedef struct {
char text[PKGMAP_PATH_BUF];
size_t used, count;
} python_members_t;
static void python_add_member(void *context, const char *member, size_t length) {
python_members_t *members = context;
if (!length || memchr(member, '\n', length) ||
members->used + length + 2 > sizeof(members->text))
return;
memcpy(members->text + members->used, member, length);
members->used += length;
members->text[members->used++] = '\n';
members->text[members->used] = '\0';
members->count++;
}
static void parse_python_manifest(const char *basename, const char *source, int source_len,
const char *rel_path, cbm_pkg_entries_t *entries) {
char *directory = path_dirname(rel_path);
if (!directory) {
entries->incomplete = true;
return;
}
python_roots_t roots = {.directory = directory, .entries = entries};
python_add_root(&roots, "src", 3);
python_add_root(&roots, ".", 1);
cbm_py_layout_parse(basename, source, (size_t)source_len, python_add_root, &roots);
node_collect(entries, "py-project", directory, basename, roots.roots);
if (!strcmp(basename, "pyproject.toml")) {
python_members_t members = {.count = 0};
cbm_py_layout_workspace_members(source, (size_t)source_len, python_add_member, &members);
if (members.count)
node_collect(entries, "py-workspace", directory, NULL, members.text);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, directory);
}
/* ── Public: manifest detection + parsing ──────────────────────── */
bool cbm_pkgmap_try_parse(const char *basename, const char *rel_path, const char *source,
@@ -1273,11 +1346,11 @@ bool cbm_pkgmap_try_parse(const char *basename, const char *rel_path, const char
parse_cargo_toml(source, source_len, rel_path, entries);
return true;
}
if (strcmp(basename, "pyproject.toml") == 0) {
if (cbm_py_layout_is_manifest(basename)) {
/* A distribution name is not an import path or a package directory.
* Do not invent competing root/src mappings from [project].name.
* Exact physical modules and explicitly supplied mappings retain
* their existing independent resolution paths. */
* Do not invent competing root/src mappings from [project].name: the
* manifest's layout is what says where modules live. */
parse_python_manifest(basename, source, source_len, rel_path, entries);
return true;
}
if (strcmp(basename, "composer.json") == 0) {
@@ -1420,6 +1493,12 @@ static void node_parent_path(char *path) {
path[0] = '\0';
}
/* The directory of a private fact of this kind, without reading the map. */
static bool python_fact_key(const cbm_pkg_entry_t *item, const char *kind, char *directory,
size_t capacity) {
return item->node_fact && node_fact_directory(item->pkg_name, kind, directory, capacity);
}
static bool node_glob_covered(CBMHashTable *map, const char *directory) {
if (!node_fact(map, "directory", directory, NULL) ||
node_fact(map, "excluded-child", directory, NULL))
@@ -1709,16 +1788,19 @@ static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_ent
excluded_count);
continue;
}
if (!is_pkgmap_manifest_basename(name))
/* setup.cfg and setup.py only ever add a source root, so one that cannot
* be read withholds that and nothing else. */
bool optional = cbm_py_layout_is_manifest(name) && !is_pkgmap_manifest_basename(name);
if (!is_pkgmap_manifest_basename(name) && !optional)
continue;
if (!info.is_regular) {
entries->incomplete = true;
entries->incomplete = entries->incomplete || !optional;
continue;
}
int source_len = 0;
char *source = pkgmap_read_file(abs_path, &source_len);
if (!source) {
entries->incomplete = true;
entries->incomplete = entries->incomplete || !optional;
continue;
}
if (cbm_pkgmap_try_parse(name, rel_path, source, source_len, entries)) {
@@ -1743,12 +1825,97 @@ static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_ent
* coverage, including symlinks / Windows reparse points inside the domain.
* This is what lets bare workspace imports (e.g. "@org/pkg" declared in
* an ignored package.json) resolve on Windows as well as POSIX. */
/* One member root's top-level names, under the workspace that lists it. Two members providing a
* name collide in the map's merge, which leaves that name to no one. */
static void python_note_member_root(const char *repo_path, const char *workspace, const char *root,
cbm_pkg_entries_t *entries) {
char absolute[PKGMAP_PATH_BUF];
int size = snprintf(absolute, sizeof(absolute), "%s%s%s", repo_path, root[0] ? "/" : "", root);
cbm_dir_t *dir = size > 0 && (size_t)size < sizeof(absolute) ? cbm_opendir(absolute) : NULL;
if (!dir)
return;
cbm_dirent_t *entry;
while ((entry = cbm_readdir(dir)) != NULL) {
char name[PKGMAP_PATH_BUF];
int length = snprintf(name, sizeof(name), "%s", entry->name);
if (length <= 0 || (size_t)length >= sizeof(name) || entry->name[0] == '.')
continue;
if (length > 3 && !strcmp(name + length - 3, ".py"))
name[length - 3] = '\0';
if (!strchr(name, '.'))
node_collect(entries, "py-member-head", workspace, name, root);
}
cbm_closedir(dir);
}
/* One project under one ancestor that declares a workspace, when that workspace lists it. */
static void python_note_member(const char *repo_path, const char *workspace, const char *project,
const char *patterns, const char *roots,
cbm_pkg_entries_t *entries) {
size_t prefix = strlen(workspace);
const char *below = prefix ? project + prefix + 1 : project;
bool member = false;
for (const char *pattern = patterns; *pattern && !member;) {
size_t length = strcspn(pattern, "\n");
char one[PKGMAP_PATH_BUF];
if (length < sizeof(one)) {
memcpy(one, pattern, length);
one[length] = '\0';
member = cbm_py_layout_member_matches(one, below);
}
pattern += length + (pattern[length] != '\0');
}
for (const char *root = roots; member && *root;) {
size_t length = strcspn(root, "\n");
char one[PKGMAP_PATH_BUF];
if (length < sizeof(one)) {
memcpy(one, root, length);
one[length] = '\0';
python_note_member_root(repo_path, workspace, one, entries);
}
root += length + (root[length] != '\0');
}
}
/* A uv workspace's members are installed together. Each member project's roots are read once,
* after the walk has seen every manifest: one pass over the collected facts, one lookup per
* ancestor of each project, plus the size of the member directories. */
static void python_finish_workspaces(const char *repo_path, cbm_pkg_entries_t *entries) {
int total = entries->count;
CBMHashTable *spaces = cbm_ht_create(8);
if (!spaces)
return;
for (int w = 0; w < total; w++) {
char workspace[PKGMAP_PATH_BUF];
if (python_fact_key(&entries->items[w], "py-workspace", workspace, sizeof(workspace)))
cbm_ht_set(spaces, entries->items[w].pkg_name, entries->items[w].entry_rel);
}
for (int p = 0; p < total && cbm_ht_count(spaces); p++) {
char project[PKGMAP_PATH_BUF], ancestor[PKGMAP_PATH_BUF], key[PKGMAP_PATH_BUF];
if (!python_fact_key(&entries->items[p], "py-project", project, sizeof(project)) ||
!project[0])
continue;
const char *roots = entries->items[p].entry_rel;
memcpy(ancestor, project, strlen(project) + 1);
do {
node_parent_path(ancestor);
const char *patterns = node_fact_key(key, sizeof(key), "py-workspace", ancestor, NULL)
? cbm_ht_get(spaces, key)
: NULL;
if (patterns)
python_note_member(repo_path, ancestor, project, patterns, roots, entries);
} while (ancestor[0]);
}
cbm_ht_free(spaces);
}
int cbm_pkgmap_scan_repo(const char *repo_path, cbm_pkg_entries_t *entries, char **excluded_dirs,
int excluded_count) {
if (!repo_path || !entries) {
return 0;
}
int parsed = pkgmap_walk_dir(repo_path, "", entries, 0, excluded_dirs, excluded_count);
python_finish_workspaces(repo_path, entries);
cbm_log_info("pkgmap.scan_repo", "manifests", pkgmap_itoa(parsed));
return parsed;
}
@@ -3707,13 +3874,16 @@ static const cbm_gbuf_node_t *python_configured_module(const cbm_pipeline_ctx_t
typedef struct {
char base[PKGMAP_PATH_BUF]; /* empty at the repo root */
char own[PKGMAP_PATH_BUF]; /* the file's top-level package, empty outside one */
/* A source root above base: the directories between are namespace packages. */
char root[PKGMAP_PATH_BUF]; /* empty when base itself is the search directory */
char root_own[PKGMAP_PATH_BUF]; /* the file's top-level name under that root */
} python_search_t;
/* Where a file's absolute imports look first. A file inside a package was loaded under
* that package, so the directory above its topmost __init__.py is on sys.path. A file
* outside any package is a script or a test, and its own directory is. */
static bool python_search_path(const cbm_pipeline_ctx_t *ctx, const char *source_rel,
python_search_t *search) {
static bool python_package_base(const cbm_pipeline_ctx_t *ctx, const char *source_rel,
python_search_t *search) {
search->own[0] = '\0';
int length = snprintf(search->base, sizeof(search->base), "%s", source_rel);
if (length <= 0 || (size_t)length >= sizeof(search->base))
@@ -3743,6 +3913,145 @@ static bool python_search_path(const cbm_pipeline_ctx_t *ctx, const char *source
}
}
/* A package without __init__.py is still one under a stated source root: poetry keeps
* src/poetry so, to share the name with poetry-core. The nearest root above the package
* base decides what the file is, however many such directories lie between. */
static bool python_search_path(const cbm_pipeline_ctx_t *ctx, const char *source_rel,
python_search_t *search) {
search->root[0] = '\0';
search->root_own[0] = '\0';
if (!python_package_base(ctx, source_rel, search))
return false;
CBMHashTable *map = cbm_pipeline_get_pkgmap();
if (!map || !search->base[0])
return true;
char directory[PKGMAP_PATH_BUF];
snprintf(directory, sizeof(directory), "%s", search->base);
if (node_fact(map, "py-root", directory, NULL))
return true;
for (node_parent_path(directory); directory[0]; node_parent_path(directory)) {
const char *fact = node_fact(map, "py-root", directory, NULL);
if (!fact || !strcmp(fact, ambiguous_pkgmap_value))
continue;
size_t size = strlen(directory);
const char *below = source_rel + size + 1;
int written = snprintf(search->root, sizeof(search->root), "%s", directory);
int own = snprintf(search->root_own, sizeof(search->root_own), "%.*s",
(int)strcspn(below, "/"), below);
return written > 0 && (size_t)written < sizeof(search->root) && own > 0 &&
(size_t)own < sizeof(search->root_own);
}
return true;
}
/* What an import names under one search directory. A module of its first name that
* lacks the rest hides anything further along. */
static const cbm_gbuf_node_t *python_under(const cbm_pipeline_ctx_t *ctx, const char *directory,
const char *normalized, int head, bool *hidden) {
char near_path[PKGMAP_PATH_BUF], head_path[PKGMAP_PATH_BUF];
int near_size = snprintf(near_path, sizeof(near_path), "%s%s%s", directory,
directory[0] ? "/" : "", normalized);
int head_size = snprintf(head_path, sizeof(head_path), "%s%s%.*s", directory,
directory[0] ? "/" : "", head, normalized);
if (near_size <= 0 || (size_t)near_size >= sizeof(near_path) || head_size <= 0 ||
(size_t)head_size >= sizeof(head_path)) {
*hidden = true;
return NULL;
}
const cbm_gbuf_node_t *found = python_module_at_path(ctx, near_path);
*hidden = !found && python_module_at_path(ctx, head_path);
return found;
}
/* A stated source root that is itself a package holds no top-level module. */
static bool python_root_is_package(const cbm_pipeline_ctx_t *ctx, const char *root, size_t size) {
char init[PKGMAP_PATH_BUF];
int written = snprintf(init, sizeof(init), "%.*s/__init__.py", (int)size, root);
return written <= 0 || (size_t)written >= sizeof(init) || python_module_file(ctx, init);
}
/* The roots one directory's manifests state. Found is the module they agree on. */
static bool python_project_roots(const cbm_pipeline_ctx_t *ctx, CBMHashTable *map,
const char *directory, const char *normalized, int head,
const cbm_gbuf_node_t **found, bool *hidden) {
const char *manifest;
for (size_t m = 0; (manifest = cbm_py_layout_manifest_at(m)) != NULL; m++) {
const char *roots = node_fact(map, "py-project", directory, manifest);
if (!roots || !strcmp(roots, ambiguous_pkgmap_value))
continue;
for (const char *root = roots; *root;) {
size_t size = strcspn(root, "\n");
char path[PKGMAP_PATH_BUF];
int written = snprintf(path, sizeof(path), "%.*s", (int)size, root);
root += size + (root[size] != '\0');
if (written <= 0 || (size_t)written >= sizeof(path) ||
python_root_is_package(ctx, path, size))
continue;
bool blocked = false;
const cbm_gbuf_node_t *module = python_under(ctx, path, normalized, head, &blocked);
if (module && *found && module != *found)
return false;
*found = module ? module : *found;
*hidden = *hidden || blocked;
}
}
return true;
}
/* Nothing nearer has the name, so it is what an installed project provides: the
* file's own project first, then each enclosing one, as an example inside a project
* depends on that project. Work is the path's depth times the fixed manifest names. */
static const cbm_gbuf_node_t *python_installed(const cbm_pipeline_ctx_t *ctx,
const char *source_rel, const char *normalized,
int head, const cbm_gbuf_node_t *rooted) {
CBMHashTable *map = cbm_pipeline_get_pkgmap();
if (!map || cbm_py_layout_is_stdlib(normalized, (size_t)head))
return rooted;
char directory[PKGMAP_PATH_BUF];
int size = snprintf(directory, sizeof(directory), "%s", source_rel);
if (size <= 0 || (size_t)size >= sizeof(directory))
return rooted;
node_parent_path(directory);
const cbm_gbuf_node_t *found = NULL;
bool hidden = false;
for (;;) {
/* The repository root is searched first whatever the file: a module there stands
* before the root manifest's own roots, as it does for a type checker. */
if (!directory[0] && rooted)
return rooted;
if (!python_project_roots(ctx, map, directory, normalized, head, &found, &hidden))
return NULL;
if (found || hidden || !directory[0])
break;
node_parent_path(directory);
}
if (!found && !hidden && !rooted) {
/* A sibling member of a uv workspace enclosing the file. */
char head_name[PKGMAP_PATH_BUF];
int written = snprintf(head_name, sizeof(head_name), "%.*s", head, normalized);
if (written > 0 && (size_t)written < sizeof(head_name)) {
snprintf(directory, sizeof(directory), "%s", source_rel);
for (node_parent_path(directory);; node_parent_path(directory)) {
const char *root = node_fact(map, "py-workspace", directory, NULL)
? node_fact(map, "py-member-head", directory, head_name)
: NULL;
if (root && strcmp(root, ambiguous_pkgmap_value)) {
found = python_under(ctx, root, normalized, head, &hidden);
break;
}
if (!directory[0])
break;
}
}
}
if (!found)
return hidden ? NULL : rooted;
return !rooted || rooted == found ? found : NULL;
}
/* An absolute import, given what it names relative to the repo root. Python takes the first
* match on sys.path, and only the file's own search directory is known. Two different
* modules leave the choice to a path we cannot see, except for the file's own package,
@@ -3755,24 +4064,30 @@ static const cbm_gbuf_node_t *python_on_search_path(const cbm_pipeline_ctx_t *ct
return rooted;
if (!python_search_path(ctx, source_rel, &search))
return NULL;
if (!search.base[0])
return rooted;
char near_path[PKGMAP_PATH_BUF], head_path[PKGMAP_PATH_BUF];
int head = (int)strcspn(normalized, "/");
int near_size = snprintf(near_path, sizeof(near_path), "%s/%s", search.base, normalized);
int head_size =
snprintf(head_path, sizeof(head_path), "%s/%.*s", search.base, head, normalized);
if (near_size <= 0 || (size_t)near_size >= sizeof(near_path) || head_size <= 0 ||
(size_t)head_size >= sizeof(head_path))
return NULL;
const cbm_gbuf_node_t *found = python_module_at_path(ctx, near_path);
if (found) {
bool own =
(int)strlen(search.own) == head && !strncmp(search.own, normalized, (size_t)head);
return !rooted || own ? found : NULL;
bool hidden = false;
if (search.root[0]) {
const cbm_gbuf_node_t *found = python_under(ctx, search.root, normalized, head, &hidden);
bool own = (int)strlen(search.root_own) == head &&
!strncmp(search.root_own, normalized, (size_t)head);
if (found)
return !rooted || own ? found : NULL;
if (hidden || own)
return NULL;
}
/* A module of that first name here hides the one at the root, whatever it lacks. */
return python_module_at_path(ctx, head_path) ? NULL : rooted;
/* Inside a package below a source root, the package's parent is not on sys.path. */
if (search.base[0] && (!search.root[0] || !search.own[0])) {
const cbm_gbuf_node_t *found = python_under(ctx, search.base, normalized, head, &hidden);
if (found) {
bool own =
(int)strlen(search.own) == head && !strncmp(search.own, normalized, (size_t)head);
return !rooted || own ? found : NULL;
}
/* A module of that first name here hides the one at the root, whatever it lacks. */
if (hidden)
return NULL;
}
return python_installed(ctx, source_rel, normalized, head, rooted);
}
static const cbm_gbuf_node_t *python_exact_module(const cbm_pipeline_ctx_t *ctx,
+317 -117
View File
@@ -13,6 +13,7 @@
#include "resolve/lookup.h"
#include "resolve/policy.h"
#include "resolve/py_members.h"
#include "resolve/rust_names.h"
#include "resolve/snapshot.h"
#include "resolve/solver.h"
@@ -117,6 +118,8 @@ typedef struct {
const char *module_qn;
cbm_resolve_proof_source_t source;
bool complete, ambiguous, python_mutated;
bool python_audited; /* every statement was seen, whatever __getattr__ answers besides */
bool python_star_chosen; /* a condition chooses among its stars: the binder stopped at one */
const char **python_names; /* module-level binders, sorted, parallel to python_name_at */
uint32_t *python_name_at;
size_t python_name_count;
@@ -146,11 +149,14 @@ typedef struct {
} python_nested_list_t;
typedef struct python_pending_import {
const char *key, *slot, *parent_file, *child_file, *member, *escape_root, *escape_slot;
const char *key, *slot, *parent_file, *child_file, *member, *write_root, *write_slot;
const cbm_resolve_proof_t *audit;
const char *loaded_module;
const cbm_resolve_proof_t *load_audit;
bool module, audited, escaped, escape_module;
const char *source_file;
const char **writes; /* dotted paths stored through this import, NULL for any name */
size_t write_count;
bool module, audited, write_module, in_parent;
struct python_pending_import *next;
} python_pending_import_t;
@@ -190,6 +196,8 @@ struct cbm_shared_resolver {
bool python_unknown_namespace_escape;
cbm_ecma_table_t *ecma;
CBMHashTable *python_star_provided; /* "<module>.<name>" a closed star import binds */
CBMHashTable *python_written; /* "<module>.<path>" a statement stores or deletes */
CBMHashTable *python_parent_loaded; /* a submodule its package's own __init__ loads */
CBMHashTable *python_nested; /* file -> python_nested_list_t */
cbm_py_members_t *python_members;
CBMHashTable *http_defaults, *http_sites, *http_unsafe;
@@ -387,10 +395,52 @@ static bool declarations_coexist(CBMLanguage language) {
}
}
/* A typing.overload stub states a signature for the def that follows; with it, one declaration.
* The binding audit proves the decorator is the real one before anything resolves to the name. */
static bool python_overload_stub_definition(const CBMDefinition *definition) {
for (const char *const *decorator = definition->decorators; decorator && *decorator;
decorator++) {
const char *dot = strrchr(*decorator, '.');
const char *name = dot ? dot + 1 : *decorator;
if (*name == '@')
name++;
if (!strcmp(name, "overload"))
return true;
}
return false;
}
/* `@x.setter def x`: the same property as the def it follows, and its one graph node keeps
* this span. */
static bool python_accessor_definition(const CBMDefinition *definition) {
size_t length = definition->name ? strlen(definition->name) : 0;
for (const char *const *decorator = definition->decorators; length && decorator && *decorator;
decorator++) {
const char *text = **decorator == '@' ? *decorator + 1 : *decorator;
if (strncmp(text, definition->name, length) || text[length] != '.')
continue;
const char *member = text + length + 1;
if (!strcmp(member, "setter") || !strcmp(member, "getter") || !strcmp(member, "deleter"))
return true;
}
return false;
}
static bool add_target(cbm_shared_resolver_t *resolver, const CBMDefinition *definition,
const char *file, CBMLanguage language) {
if (!definition->qualified_name || !definition->label || !file)
return true;
if (language == CBM_LANG_PYTHON && python_overload_stub_definition(definition))
return true;
if (language == CBM_LANG_PYTHON && python_accessor_definition(definition)) {
target_proof_t *property = cbm_ht_get(resolver->targets, definition->qualified_name);
if (property && !strcmp(property->file, file) &&
!strcmp(property->label, definition->label)) {
property->start_line = definition->start_line;
property->end_line = definition->end_line;
return true;
}
}
target_proof_t *prior = cbm_ht_get(resolver->targets, definition->qualified_name);
if (prior && prior->rust && language == CBM_LANG_RUST) {
prior->ambiguous |= strcmp(prior->file, file) != 0;
@@ -471,11 +521,16 @@ static bool owner_kind(TSNode node, const char *kind) {
}
static bool note_owner_branch(cbm_shared_resolver_t *resolver, const CBMFileResult *result,
const char *file, const char *source, size_t length, TSNode group,
TSNode block, unsigned branch) {
uint32_t count = ts_node_named_child_count(block);
for (uint32_t i = 0; i < count; i++) {
TSNode function = ts_node_named_child(block, i);
const char *file, const char *source, size_t length,
const char *scope, TSNode group, TSNode block, unsigned branch) {
TSTreeCursor cursor = ts_tree_cursor_new(block);
bool ok = true;
for (bool more = ts_tree_cursor_goto_first_child(&cursor); ok && more;
more = ts_tree_cursor_goto_next_sibling(&cursor)) {
TSNode statement = ts_tree_cursor_current_node(&cursor);
TSNode function = owner_kind(statement, "decorated_definition")
? owner_field(statement, "definition")
: statement;
TSNode name = owner_field(function, "name"), body = owner_field(function, "body");
if (!owner_kind(function, "function_definition") || !owner_kind(name, "identifier") ||
!owner_kind(body, "block"))
@@ -483,54 +538,115 @@ static bool note_owner_branch(cbm_shared_resolver_t *resolver, const CBMFileResu
uint32_t start = ts_node_start_byte(name), end = ts_node_end_byte(name);
if (end <= start || end > length || end - start > INT_MAX)
continue;
const char *qn = cbm_arena_sprintf(&resolver->arena, "%s.%.*s", result->module_qn,
(int)(end - start), source + start);
if (!qn)
return false;
const char *qn = cbm_arena_sprintf(&resolver->arena, "%s.%.*s", scope, (int)(end - start),
source + start);
if (!qn) {
ok = false;
break;
}
target_proof_t *target = cbm_ht_get(resolver->targets, qn);
if (!target || !target->ambiguous || target->declarations != 2 ||
strcmp(target->file, file) || strcmp(target->label, "Function") ||
strcmp(target->file, file) ||
(strcmp(target->label, "Function") && strcmp(target->label, "Method")) ||
target->start_line == target->second_start_line)
continue;
uint32_t first = ts_node_start_point(function).row + 1;
uint32_t outer = ts_node_start_point(statement).row + 1;
uint32_t last = ts_node_end_point(function).row + 1;
unsigned member;
if (first == target->start_line && last == target->end_line)
if ((first == target->start_line || outer == target->start_line) &&
last == target->end_line)
member = 0;
else if (first == target->second_start_line && last == target->second_end_line)
else if ((first == target->second_start_line || outer == target->second_start_line) &&
last == target->second_end_line)
member = 1;
else
continue;
if (!target->conditional) {
conditional_owner_t *owner = cbm_arena_calloc(&resolver->arena, sizeof(*owner));
if (!owner)
return false;
if (!owner) {
ok = false;
break;
}
owner->content = cbm_arena_strdup(&resolver->arena,
result->source_content_id[CBM_SOURCE_ORIGIN_RAW]);
owner->configuration =
cbm_arena_strdup(&resolver->arena, result->source_configuration_id);
if (!owner->content || !owner->configuration)
return false;
if (!owner->content || !owner->configuration) {
ok = false;
break;
}
target->conditional = owner;
}
conditional_owner_t *owner = target->conditional;
owner->invalid |= owner->members[member].branch != 0;
owner->members[member] = (owner_variant_t){.start_line = first,
.end_line = last,
.start_byte = ts_node_start_byte(function),
.end_byte = ts_node_end_byte(function),
.body_start = ts_node_start_byte(body),
.body_end = ts_node_end_byte(body),
.group_start = ts_node_start_byte(group),
.group_end = ts_node_end_byte(group),
.branch = branch};
owner->members[member] = (owner_variant_t){
.start_line =
first == target->start_line || first == target->second_start_line ? first : outer,
.end_line = last,
.start_byte = ts_node_start_byte(function),
.end_byte = ts_node_end_byte(function),
.body_start = ts_node_start_byte(body),
.body_end = ts_node_end_byte(body),
.group_start = ts_node_start_byte(group),
.group_end = ts_node_end_byte(group),
.branch = branch};
}
return true;
ts_tree_cursor_delete(&cursor);
return ok;
}
/* An if/else whose two arms each declare a name: one declaration of it, whichever arm runs. */
static bool note_owner_group(cbm_shared_resolver_t *resolver, const CBMFileResult *result,
const char *file, const char *source, size_t length, const char *scope,
TSNode group) {
TSNode first = owner_field(group, "consequence");
TSNode alternative = owner_field(group, "alternative");
TSNode second = owner_field(alternative, "body");
if (!owner_kind(first, "block") || !owner_kind(alternative, "else_clause") ||
!owner_kind(second, "block"))
return true;
return note_owner_branch(resolver, result, file, source, length, scope, group, first, 1) &&
note_owner_branch(resolver, result, file, source, length, scope, group, second, 2);
}
/* Audit one raw syntax tree in linear time: each if/else directly in a scope's body, where the
* scope is the module or a def or class reached through bodies. elif, nested groups, other
* languages and cross-file build variants remain quality gaps. */
static bool note_conditional_owners_in(cbm_shared_resolver_t *resolver, const CBMFileResult *result,
const char *file, const char *source, size_t length,
const char *scope, TSNode body, unsigned depth) {
/* A cursor, not an index loop: a fetch by index walks from the first child. */
bool ok = depth < 32;
TSTreeCursor cursor = ts_tree_cursor_new(body);
for (bool more = ok && ts_tree_cursor_goto_first_child(&cursor); ok && more;
more = ts_tree_cursor_goto_next_sibling(&cursor)) {
TSNode statement = ts_tree_cursor_current_node(&cursor);
if (owner_kind(statement, "if_statement")) {
ok = note_owner_group(resolver, result, file, source, length, scope, statement);
continue;
}
TSNode definition = owner_kind(statement, "decorated_definition")
? owner_field(statement, "definition")
: statement;
if (!owner_kind(definition, "function_definition") &&
!owner_kind(definition, "class_definition"))
continue;
TSNode name = owner_field(definition, "name"), inner = owner_field(definition, "body");
if (!owner_kind(name, "identifier") || !owner_kind(inner, "block"))
continue;
uint32_t start = ts_node_start_byte(name), end = ts_node_end_byte(name);
if (end <= start || end > length)
continue;
const char *child = cbm_arena_sprintf(&resolver->arena, "%s.%.*s", scope,
(int)(end - start), source + start);
ok = child && note_conditional_owners_in(resolver, result, file, source, length, child,
inner, depth + 1);
}
ts_tree_cursor_delete(&cursor);
return ok;
}
/* Audit one raw syntax tree in linear time. This deliberately supports only
* direct module-level if/else bodies. elif, nested groups, decorated functions,
* other languages and cross-file build variants remain quality gaps. */
static bool note_conditional_owners(cbm_shared_resolver_t *resolver, const CBMFileResult *result,
const char *file, CBMLanguage language, const char *source,
size_t length) {
@@ -541,26 +657,8 @@ static bool note_conditional_owners(cbm_shared_resolver_t *resolver, const CBMFi
TSNode root = ts_tree_root_node(result->cached_tree);
if (!owner_kind(root, "module") || ts_node_has_error(root))
return true;
/* A cursor, not an index loop: a fetch by index walks from the first child. */
bool ok = true;
TSTreeCursor cursor = ts_tree_cursor_new(root);
if (ts_tree_cursor_goto_first_child(&cursor)) {
do {
TSNode group = ts_tree_cursor_current_node(&cursor);
if (!owner_kind(group, "if_statement"))
continue;
TSNode first = owner_field(group, "consequence");
TSNode alternative = owner_field(group, "alternative");
TSNode second = owner_field(alternative, "body");
if (!owner_kind(first, "block") || !owner_kind(alternative, "else_clause") ||
!owner_kind(second, "block"))
continue;
ok = note_owner_branch(resolver, result, file, source, length, group, first, 1) &&
note_owner_branch(resolver, result, file, source, length, group, second, 2);
} while (ok && ts_tree_cursor_goto_next_sibling(&cursor));
}
ts_tree_cursor_delete(&cursor);
return ok;
return note_conditional_owners_in(resolver, result, file, source, length, result->module_qn,
root, 0);
}
static bool target_materialized(const file_resolver_t *resolver, const cbm_gbuf_node_t *node) {
@@ -951,8 +1049,13 @@ static bool capture_python_stars(cbm_shared_resolver_t *resolver, const cbm_pipe
entry->python_stars = cbm_arena_calloc(&resolver->arena, count * sizeof(*entry->python_stars));
if (!entry->python_stars)
return false;
size_t kept = 0;
for (size_t i = 0; i < count; i++) {
const cbm_resolve_python_star_t *star = cbm_resolve_binding_python_star_at(index, i);
/* Under TYPE_CHECKING it binds nothing at run time, and only some modules have another
* way to have those names. */
if (star->annotation_only && !cbm_resolve_binding_python_names_come_at_run_time(index))
continue;
CBMImport path = {.local_name = star->module_path,
.module_path = star->module_path,
.kind = CBM_IMPORT_KIND_PYTHON_MODULE_ALIAS};
@@ -964,9 +1067,9 @@ static bool capture_python_stars(cbm_shared_resolver_t *resolver, const cbm_pipe
cbm_free(CBM_MEM_CLASS_SEMANTIC, slot);
if (lost)
return false;
entry->python_stars[i] = (python_star_source_t){.module = copy, .at = star->at};
entry->python_stars[kept++] = (python_star_source_t){.module = copy, .at = star->at};
}
entry->python_star_count = count;
entry->python_star_count = kept;
return true;
}
@@ -995,6 +1098,12 @@ static bool capture_python_namespace(cbm_shared_resolver_t *resolver, const cbm_
entry->source = context.source;
entry->module_qn = module;
entry->complete = cbm_resolve_binding_python_namespace_closed(index);
entry->python_audited =
cbm_resolve_binding_status(index) &&
!strcmp(cbm_resolve_binding_status(index), "binding_complete_supported_subset");
entry->python_star_chosen =
cbm_resolve_binding_status(index) &&
!strcmp(cbm_resolve_binding_status(index), "conditional_wildcard_import");
size_t count = cbm_resolve_binding_python_namespace_size(index);
if (count > SIZE_MAX / sizeof(*entry->python_names))
return false;
@@ -1070,9 +1179,55 @@ static bool python_loaded_module(cbm_shared_resolver_t *resolver, const cbm_pipe
return *out != NULL;
}
/* What this file writes through the module the import bound. */
static bool python_note_import_writes(cbm_shared_resolver_t *resolver,
const cbm_pipeline_ctx_t *ctx,
const cbm_resolve_binding_index_t *index, const char *file,
const CBMImport *import, python_pending_import_t *entry) {
const char *member = NULL;
while (cbm_resolve_binding_python_import_write(index, import->local_name, entry->write_count,
&member))
entry->write_count++;
if (!entry->write_count)
return true;
entry->writes = cbm_arena_alloc(&resolver->arena, entry->write_count * sizeof(*entry->writes));
if (!entry->writes)
return false;
for (size_t i = 0; i < entry->write_count; i++) {
cbm_resolve_binding_python_import_write(index, import->local_name, i, &member);
entry->writes[i] = member ? cbm_arena_strdup(&resolver->arena, member) : NULL;
if (member && !entry->writes[i])
return false;
}
char *effect = cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, import,
&entry->write_module);
if (effect) {
entry->write_slot = cbm_arena_strdup(&resolver->arena, effect);
cbm_free(CBM_MEM_CLASS_SEMANTIC, effect);
if (!entry->write_slot)
return false;
}
if (import->kind == CBM_IMPORT_KIND_PYTHON_MODULE_ROOT && import->local_name) {
CBMImport root = *import;
root.module_path = import->local_name;
root.kind = CBM_IMPORT_KIND_PYTHON_MODULE_ALIAS;
bool is_module = false;
char *bound =
cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, &root, &is_module);
if (bound) {
entry->write_root = cbm_arena_strdup(&resolver->arena, bound);
cbm_free(CBM_MEM_CLASS_SEMANTIC, bound);
if (!entry->write_root)
return false;
}
}
return true;
}
static bool defer_python_import(cbm_shared_resolver_t *resolver, const cbm_pipeline_ctx_t *ctx,
const char *file, const CBMImport *import, const char *key,
const char *slot, bool module, bool audited, bool escaped,
const cbm_resolve_binding_index_t *index, const char *file,
const CBMImport *import, const char *key, const char *slot,
bool module, bool audited, bool runs,
const cbm_resolve_proof_t *audit, const char *loaded_module,
const cbm_resolve_proof_t *load_audit) {
python_pending_import_t *entry = cbm_arena_calloc(&resolver->arena, sizeof(*entry));
@@ -1082,36 +1237,14 @@ static bool defer_python_import(cbm_shared_resolver_t *resolver, const cbm_pipel
entry->slot = slot ? cbm_arena_strdup(&resolver->arena, slot) : NULL;
entry->module = module;
entry->audited = audited;
entry->escaped = escaped;
entry->audit = audit;
entry->loaded_module = loaded_module;
entry->load_audit = load_audit;
if (!entry->key || (slot && !entry->slot))
entry->source_file = cbm_arena_strdup(&resolver->arena, file);
if (!entry->key || (slot && !entry->slot) || !entry->source_file)
return false;
if (runs && !python_note_import_writes(resolver, ctx, index, file, import, entry))
return false;
if (escaped) {
char *effect = cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, import,
&entry->escape_module);
if (effect) {
entry->escape_slot = cbm_arena_strdup(&resolver->arena, effect);
cbm_free(CBM_MEM_CLASS_SEMANTIC, effect);
if (!entry->escape_slot)
return false;
}
}
if (escaped && import->kind == CBM_IMPORT_KIND_PYTHON_MODULE_ROOT && import->local_name) {
CBMImport root = *import;
root.module_path = import->local_name;
root.kind = CBM_IMPORT_KIND_PYTHON_MODULE_ALIAS;
bool is_module = false;
char *bound =
cbm_pipeline_python_import_slot(ctx->gbuf, ctx->project_name, file, &root, &is_module);
if (bound) {
entry->escape_root = cbm_arena_strdup(&resolver->arena, bound);
cbm_free(CBM_MEM_CLASS_SEMANTIC, bound);
if (!entry->escape_root)
return false;
}
}
const cbm_gbuf_node_t *parent, *child;
if (audited && !module &&
cbm_pipeline_python_submodule_candidate(ctx->gbuf, ctx->project_name, file, import, &parent,
@@ -1119,6 +1252,8 @@ static bool defer_python_import(cbm_shared_resolver_t *resolver, const cbm_pipel
entry->parent_file = cbm_arena_strdup(&resolver->arena, parent->file_path);
entry->child_file = cbm_arena_strdup(&resolver->arena, child->file_path);
entry->member = cbm_arena_strdup(&resolver->arena, strrchr(import->module_path, '.') + 1);
entry->in_parent = !strcmp(parent->file_path, file) && import->local_name &&
!strcmp(import->local_name, entry->member);
if (!entry->parent_file || !entry->child_file || !entry->member)
return false;
}
@@ -1511,10 +1646,8 @@ static bool python_export_import(cbm_shared_resolver_t *resolver, const cbm_pipe
"python_explicit_module_load_barrier", import)
: NULL;
ok = ok && (!loaded || load_audit) &&
defer_python_import(
resolver, ctx, file, import, fact_key, slot, module, audited,
runs && cbm_resolve_binding_python_import_escaped(index, import->local_name), audit,
loaded, load_audit);
defer_python_import(resolver, ctx, index, file, import, fact_key, slot, module, audited,
runs, audit, loaded, load_audit);
/* A plain dotted import binds only a package root, not the loaded leaf
* under that root's exported name. Exact module-path uses still retain
* the private import fact above. */
@@ -1544,14 +1677,14 @@ static const char *python_member_import_key(void *context, CBMArena *arena,
const cbm_resolve_python_nested_import_t *nested,
const char *chain, const char **rest) {
python_member_imports_t *lookup = context;
size_t dotted = strcspn(chain, "(");
size_t dotted = strcspn(chain, "([@");
if (nested) {
const char *base = nested->import.kind == CBM_IMPORT_KIND_PYTHON_MODULE_ROOT
? nested->import.module_path
: nested->import.local_name;
size_t length = strlen(base);
if (length > dotted || strncmp(chain, base, length) ||
(chain[length] && chain[length] != '.' && chain[length] != '('))
(chain[length] && !strchr(".([@", chain[length])))
return NULL;
*rest = chain + length;
return python_nested_key(arena, lookup->file, nested->at, nested->import.local_name);
@@ -1692,8 +1825,9 @@ static bool export_file(cbm_shared_resolver_t *resolver, const cbm_pipeline_ctx_
const CBMDefinition *definition = &result->defs.items[d];
/* Indexing child.py does not execute it or overwrite an explicit child
* value in __init__.py. Actual loads are separate effect observations. */
if (file->language == CBM_LANG_PYTHON && definition->label &&
!strcmp(definition->label, "Module"))
if (file->language == CBM_LANG_PYTHON &&
((definition->label && !strcmp(definition->label, "Module")) ||
python_overload_stub_definition(definition) || python_accessor_definition(definition)))
continue;
const char *qn = definition->qualified_name;
if (qn && qn[0]) {
@@ -1737,7 +1871,12 @@ static bool export_file(cbm_shared_resolver_t *resolver, const cbm_pipeline_ctx_
python_export_import(
resolver, ctx, result, file->rel_path, &arena, index, &item->import, key,
cbm_resolve_binding_python_nested_audited(index, item),
item->scope == 0 && !item->annotation_only, !item->annotation_only);
/* A TYPE_CHECKING import runs nothing. In a module whose names can come at
* run time it is what the module states that name to be: celery and every
* PEP 562 lazy package rely on it. Any other module has no such name then. */
item->scope == 0 && (!item->annotation_only ||
cbm_resolve_binding_python_names_come_at_run_time(index)),
!item->annotation_only);
}
if (ok && file->language == CBM_LANG_PYTHON && cbm_resolve_binding_status(index) &&
!strcmp(cbm_resolve_binding_status(index), "binding_complete_supported_subset")) {
@@ -1773,14 +1912,24 @@ static int compare_export(const void *left, const void *right) {
return strcmp((*(export_proof_t *const *)left)->key, (*(export_proof_t *const *)right)->key);
}
/* `from pkg import child` finds the submodule when pkg's namespace has no such name. In
* pkg's own __init__ the statement is itself that name's binder, and it runs before the
* name exists, so a binder there of its own does not count. */
static bool python_namespace_absent(cbm_shared_resolver_t *resolver,
const namespace_proof_t *parent, const char *name) {
if (!parent || !parent->complete || parent->ambiguous || parent->python_mutated || !name ||
const namespace_proof_t *parent, const char *name,
bool in_parent) {
if (!parent || parent->ambiguous || parent->python_mutated || !name ||
parent->python_open_barrier)
return false;
/* A closed star import binds this name as a value, not the submodule. */
const char *qualified = cbm_arena_sprintf(&resolver->arena, "%s.%s", parent->module_qn, name);
if (!qualified || cbm_ht_get(resolver->python_star_provided, qualified))
if (!qualified || cbm_ht_get(resolver->python_star_provided, qualified) ||
cbm_ht_get(resolver->python_written, qualified))
return false;
/* __getattr__ answers only for a name the module lacks. A child its own __init__ loads is
* set on it by that load, so it never gets to answer for this one. */
if (!parent->complete &&
!(parent->python_audited && cbm_ht_get(resolver->python_parent_loaded, qualified)))
return false;
size_t low = 0, high = parent->python_name_count;
while (low < high) {
@@ -1790,7 +1939,11 @@ static bool python_namespace_absent(cbm_shared_resolver_t *resolver,
else
high = middle;
}
return low == parent->python_name_count || strcmp(parent->python_names[low], name) != 0;
size_t binders = 0;
while (low + binders < parent->python_name_count &&
!strcmp(parent->python_names[low + binders], name))
binders++;
return binders == 0 || (in_parent && binders == 1);
}
static const cbm_resolve_proof_t *python_submodule_proof(cbm_shared_resolver_t *resolver,
@@ -1838,13 +1991,36 @@ static void invalidate_python_namespace(cbm_shared_resolver_t *resolver, const c
* same-leaf files never invalidate one another. Work follows imported
* namespace prefixes, using hash lookups rather than a files-by-import scan. */
static bool collect_python_module_loads(cbm_shared_resolver_t *resolver) {
/* `from . import child` in the parent's own __init__ binds the very module a load installs
* there, so that load competes with nothing. */
CBMHashTable *own_children = cbm_ht_create(32);
if (!own_children)
return false;
for (const python_pending_import_t *import = resolver->python_imports; import;
import = import->next) {
/* A load by the parent's own __init__: a from-import of the child, or of something in it.
*/
const namespace_proof_t *source = cbm_ht_get(resolver->namespaces, import->source_file);
const char *dot = import->loaded_module ? strrchr(import->loaded_module, '.') : NULL;
if (source && dot && strlen(source->module_qn) == (size_t)(dot - import->loaded_module) &&
!strncmp(source->module_qn, import->loaded_module,
(size_t)(dot - import->loaded_module)))
cbm_ht_set(resolver->python_parent_loaded, import->loaded_module, resolver);
const char *key = import->in_parent && import->slot ? import->slot : NULL;
if (key) {
cbm_ht_set(own_children, key, resolver);
cbm_ht_set(resolver->python_parent_loaded, key, resolver);
}
}
for (const python_pending_import_t *import = resolver->python_imports; import;
import = import->next) {
if (!import->loaded_module)
continue;
char *module = cbm_arena_strdup(&resolver->arena, import->loaded_module);
if (!module || !import->load_audit)
if (!module || !import->load_audit) {
cbm_ht_free(own_children);
return false;
}
for (;;) {
const namespace_proof_t *loaded = cbm_ht_get(resolver->python_namespaces, module);
target_proof_t *declaration = cbm_ht_get(resolver->targets, module);
@@ -1855,9 +2031,12 @@ static bool collect_python_module_loads(cbm_shared_resolver_t *resolver) {
/* Keep actual load support beside any value it can overwrite. A
* source namespace alone is never a competing value declaration. */
if (loaded && cbm_ht_get(resolver->exports, module) &&
!cbm_ht_get(own_children, module) &&
!add_export(resolver, module, loaded->source.file, 0, false, NULL,
import->load_audit))
import->load_audit)) {
cbm_ht_free(own_children);
return false;
}
/* The load installs this one child on its parent: the same module
* identity a submodule import names, so it closes no other name. */
char *dot = strrchr(module, '.');
@@ -1866,21 +2045,35 @@ static bool collect_python_module_loads(cbm_shared_resolver_t *resolver) {
*dot = '\0';
}
}
cbm_ht_free(own_children);
return true;
}
static void collect_python_namespace_effects(cbm_shared_resolver_t *resolver) {
/* A statement that stores through an imported module changes the name it spells, and
* no other: `django.__file__ = None` in a test says nothing about django.forms. Only a
* write that can reach any name takes the module's whole absence proof. */
static bool collect_python_namespace_effects(cbm_shared_resolver_t *resolver) {
for (const python_pending_import_t *import = resolver->python_imports; import;
import = import->next) {
if (!import->escaped)
continue;
/* A module object handed to unknown code can be written through, so
* its own namespace loses the absence proof. A value escaping does not
* reach any other module's namespace. */
invalidate_python_namespace(resolver, import->escape_root);
if (import->escape_module)
invalidate_python_namespace(resolver, import->escape_slot);
const char *base = import->write_root ? import->write_root : import->write_slot;
for (size_t i = 0; i < import->write_count; i++) {
if (!import->writes[i]) {
invalidate_python_namespace(resolver, import->write_root);
if (import->write_module)
invalidate_python_namespace(resolver, import->write_slot);
continue;
}
if (!base)
continue;
const char *key = cbm_arena_sprintf(&resolver->arena, "%s.%s", base, import->writes[i]);
if (!key)
return false;
cbm_ht_set(resolver->python_written, key, resolver);
if (cbm_ht_get(resolver->python_written, key) != resolver)
return false;
}
}
return true;
}
/* The names `from module import *` binds. Unknown when __all__ is not a static
@@ -1888,15 +2081,16 @@ static void collect_python_namespace_effects(cbm_shared_resolver_t *resolver) {
static void python_star_interface(cbm_shared_resolver_t *resolver, namespace_proof_t *module) {
if (module->python_visible_state == 2)
return;
if (module->python_visible_state == 1 || !module->complete || module->ambiguous ||
module->python_all_unknown) {
/* A computed __all__ can only narrow what the module binds, and a file that asks a star for
* a name relies on it being there. So the public names stand in for it. */
if (module->python_visible_state == 1 || !module->complete || module->ambiguous) {
module->python_visible_open = true;
if (module->python_visible_state == 0)
module->python_visible_state = 2;
return;
}
module->python_visible_state = 1;
if (module->python_all_literal) {
if (module->python_all_literal && !module->python_all_unknown) {
module->python_visible = module->python_all;
module->python_visible_count = module->python_all_count;
module->python_visible_state = 2;
@@ -1955,7 +2149,9 @@ static bool python_explicit_after(const namespace_proof_t *module, const char *n
}
/* Last star wins; a later explicit binder wins over any star. An unknown star
* can rebind whatever precedes it, so that stops the walk. */
* can rebind whatever precedes it, so that stops the walk, and so does a star under a condition:
* the binder stops at the first of `if c: from .a import * else: from .b import *` and never
* sees the other. */
static bool python_expand_module_stars(cbm_shared_resolver_t *resolver, namespace_proof_t *module) {
for (size_t s = module->python_star_count; s-- > 0;) {
const python_star_source_t *star = &module->python_stars[s];
@@ -1963,7 +2159,7 @@ static bool python_expand_module_stars(cbm_shared_resolver_t *resolver, namespac
star->module ? cbm_ht_get(resolver->python_namespaces, star->module) : NULL;
if (source)
python_star_interface(resolver, source);
if (!source || source->python_visible_open) {
if (!source || source->python_visible_open || (module->python_star_chosen)) {
module->python_open_barrier = star->at;
for (size_t n = 0; n < module->python_name_count; n++) {
if (module->python_name_at[n] >= star->at)
@@ -2178,8 +2374,7 @@ static bool decide_python_decorated(cbm_shared_resolver_t *resolver) {
static bool settle_python_imports(cbm_shared_resolver_t *resolver) {
if (!collect_python_module_loads(resolver))
return false;
collect_python_namespace_effects(resolver);
if (!expand_python_stars(resolver))
if (!collect_python_namespace_effects(resolver) || !expand_python_stars(resolver))
return false;
for (python_pending_import_t *import = resolver->python_imports; import;
import = import->next) {
@@ -2187,8 +2382,8 @@ static bool settle_python_imports(cbm_shared_resolver_t *resolver) {
import->parent_file ? cbm_ht_get(resolver->namespaces, import->parent_file) : NULL;
const namespace_proof_t *child =
import->child_file ? cbm_ht_get(resolver->namespaces, import->child_file) : NULL;
if (child && !child->ambiguous && !child->python_mutated &&
python_namespace_absent(resolver, parent, import->member)) {
if (child && !child->ambiguous &&
python_namespace_absent(resolver, parent, import->member, import->in_parent)) {
const cbm_resolve_proof_t *proof =
python_submodule_proof(resolver, import, parent, child);
if (!proof)
@@ -2525,8 +2720,10 @@ static bool python_env_module_member(const void *context, CBMArena *arena, const
/* An attribute no module-level binder provides reads the loaded submodule. */
namespace_proof_t *parent = cbm_ht_get(resolver->python_namespaces, module);
namespace_proof_t *child = cbm_ht_get(resolver->python_namespaces, member);
if (binding.proof || !child || child->ambiguous || !parent || !parent->complete ||
parent->ambiguous || parent->python_open_barrier)
if (binding.proof || !child || child->ambiguous || !parent || parent->ambiguous ||
parent->python_open_barrier ||
(!parent->complete &&
!(parent->python_audited && cbm_ht_get(resolver->python_parent_loaded, member))))
return false;
*target = child->module_qn;
*is_module = true;
@@ -2704,6 +2901,8 @@ cbm_shared_resolver_t *cbm_shared_resolver_build(const cbm_pipeline_ctx_t *ctx,
resolver->python_queries = cbm_ht_create(128);
resolver->python_namespaces = cbm_ht_create(128);
resolver->python_star_provided = cbm_ht_create(128);
resolver->python_written = cbm_ht_create(128);
resolver->python_parent_loaded = cbm_ht_create(128);
resolver->python_nested = cbm_ht_create(128);
resolver->python_members = cbm_py_members_new();
resolver->http_defaults = cbm_ht_create(128);
@@ -2724,7 +2923,8 @@ cbm_shared_resolver_t *cbm_shared_resolver_build(const cbm_pipeline_ctx_t *ctx,
resolver->complete = true;
if (!resolver->exports || !resolver->targets || !resolver->namespaces ||
!resolver->python_queries || !resolver->python_namespaces ||
!resolver->python_star_provided || !resolver->python_nested || !resolver->python_members ||
!resolver->python_star_provided || !resolver->python_written ||
!resolver->python_parent_loaded || !resolver->python_nested || !resolver->python_members ||
!resolver->http_defaults || !resolver->http_sites || !resolver->http_unsafe ||
!resolver->http_candidates || !resolver->http_veto || !resolver->project ||
!resolver->solver || !resolver->go_types || !resolver->ecma || !resolver->source_table) {
+621
View File
@@ -0,0 +1,621 @@
#include "pipeline/py_layout.h"
#include <ctype.h>
#include <string.h>
enum { PY_LAYOUT_SECTION_MAX = 64 };
bool cbm_py_layout_is_manifest(const char *basename) {
return basename && (!strcmp(basename, "pyproject.toml") || !strcmp(basename, "setup.cfg") ||
!strcmp(basename, "setup.py") || !strcmp(basename, "tox.ini") ||
!strcmp(basename, "pytest.ini"));
}
const char *cbm_py_layout_manifest_at(size_t index) {
static const char *const names[] = {"pyproject.toml", "setup.cfg", "setup.py", "tox.ini",
"pytest.ini"};
return index < sizeof(names) / sizeof(names[0]) ? names[index] : NULL;
}
static const char *py_past_spaces(const char *p, const char *end) {
while (p < end && isspace((unsigned char)*p))
p++;
return p;
}
static const char *py_line_end(const char *p, const char *end) {
const char *newline = p < end ? memchr(p, '\n', (size_t)(end - p)) : NULL;
return newline ? newline : end;
}
static const char *py_skip_blank(const char *p, const char *end) {
while (p < end && (*p == ' ' || *p == '\t' || *p == '\r'))
p++;
return p;
}
/* ── pyproject.toml ──────────────────────────────────────────────────── */
typedef enum {
PY_FORM_STRINGS, /* every string in the value is a root */
PY_FORM_PARENTS, /* every string names a package: its directory is the root */
PY_FORM_AFTER_FROM, /* poetry's { include = "x", from = "lib" } */
PY_FORM_AFTER_EMPTY /* setuptools' { "" = "src" } */
} py_form_t;
typedef struct {
const char *section, *key;
py_form_t form;
} py_toml_key_t;
static const py_toml_key_t py_toml_keys[] = {
{"tool.setuptools.packages.find", "where", PY_FORM_STRINGS},
{"tool.setuptools", "package-dir", PY_FORM_AFTER_EMPTY},
{"tool.setuptools.package-dir", "\"\"", PY_FORM_STRINGS},
{"tool.poetry", "packages", PY_FORM_AFTER_FROM},
{"tool.poetry.packages", "from", PY_FORM_STRINGS},
{"tool.hatch.build.targets.wheel", "packages", PY_FORM_PARENTS},
{"tool.hatch.build", "packages", PY_FORM_PARENTS},
{"tool.hatch.build.targets.wheel", "sources", PY_FORM_STRINGS},
{"tool.hatch.build", "sources", PY_FORM_STRINGS},
{"tool.pdm.build", "package-dir", PY_FORM_STRINGS},
{"tool.maturin", "python-source", PY_FORM_STRINGS},
{"tool.uv.build-backend", "module-root", PY_FORM_STRINGS},
{"tool.pytest.ini_options", "pythonpath", PY_FORM_STRINGS},
{"tool.ruff", "src", PY_FORM_STRINGS},
{"tool.pyright", "extraPaths", PY_FORM_STRINGS},
{"tool.mypy", "mypy_path", PY_FORM_STRINGS},
};
/* The end of a value: its line, or past the bracket that closes it. Strings and
* comments cannot close one. */
static const char *py_toml_value_end(const char *value, const char *end) {
if (value >= end || (*value != '[' && *value != '{'))
return py_line_end(value, end);
int depth = 0;
char quote = 0;
for (const char *p = value; p < end; p++) {
if (quote) {
quote = *p == quote ? 0 : (*p == '\n' ? 0 : quote);
continue;
}
if (*p == '"' || *p == '\'') {
quote = *p;
} else if (*p == '#') {
p = py_line_end(p, end);
if (p >= end)
break;
} else if (*p == '[' || *p == '{') {
depth++;
} else if ((*p == ']' || *p == '}') && --depth == 0) {
return p + 1;
}
}
return end;
}
/* The strings of a value, in order, comments skipped. */
static void py_toml_strings(const char *value, const char *end, py_form_t form,
cbm_py_layout_emit_fn emit, void *context) {
bool after_key = false;
for (const char *p = value; p < end; p++) {
if (*p == '#') {
p = py_line_end(p, end);
if (p >= end)
return;
continue;
}
if (form == PY_FORM_AFTER_FROM && end - p >= 4 && !memcmp(p, "from", 4)) {
after_key = true;
p += 3;
continue;
}
if (*p != '"' && *p != '\'')
continue;
const char *close = p + 1;
while (close < end && *close != *p && *close != '\n')
close++;
if (close >= end || *close != *p)
return;
size_t length = (size_t)(close - p - 1);
const char *text = p + 1;
p = close;
switch (form) {
case PY_FORM_STRINGS:
emit(context, text, length);
break;
case PY_FORM_PARENTS: {
size_t cut = length;
while (cut && text[cut - 1] != '/')
cut--;
if (cut > 1)
emit(context, text, cut - 1);
break;
}
case PY_FORM_AFTER_FROM:
if (after_key)
emit(context, text, length);
after_key = false;
break;
case PY_FORM_AFTER_EMPTY:
if (after_key)
emit(context, text, length);
after_key = !length;
break;
}
}
}
static bool py_toml_section(const char *line, const char *line_end, char *section, size_t size) {
if (line >= line_end || *line != '[')
return false;
const char *p = line;
while (p < line_end && *p == '[')
p++;
const char *close = p;
while (close < line_end && *close != ']')
close++;
size_t length = (size_t)(close - p);
if (length >= size) {
section[0] = '\0';
return true;
}
memcpy(section, p, length);
section[length] = '\0';
return true;
}
static void py_parse_toml(const char *source, const char *end, cbm_py_layout_emit_fn emit,
void *context) {
char section[PY_LAYOUT_SECTION_MAX] = "";
for (const char *line = source; line < end;) {
const char *start = py_skip_blank(line, end);
const char *line_end = py_line_end(start, end);
const char *next = line_end < end ? line_end + 1 : end;
if (py_toml_section(start, line_end, section, sizeof(section))) {
line = next;
continue;
}
for (size_t i = 0; i < sizeof(py_toml_keys) / sizeof(py_toml_keys[0]); i++) {
const py_toml_key_t *key = &py_toml_keys[i];
size_t length = strlen(key->key);
if (strcmp(section, key->section) || (size_t)(line_end - start) < length ||
memcmp(start, key->key, length))
continue;
const char *value = py_skip_blank(start + length, line_end);
if (value >= line_end || *value != '=')
continue;
value = py_skip_blank(value + 1, line_end);
const char *value_end = py_toml_value_end(value, end);
if (value < line_end && (*value != '{' || key->form == PY_FORM_AFTER_EMPTY))
py_toml_strings(value, value_end, key->form, emit, context);
if (value_end > next)
next = py_line_end(value_end, end) < end ? py_line_end(value_end, end) + 1 : end;
break;
}
line = next;
}
}
/* ── setup.cfg ───────────────────────────────────────────────────────── */
static void py_cfg_tokens(const char *p, const char *end, cbm_py_layout_emit_fn emit,
void *context) {
while (p < end) {
while (p < end && isspace((unsigned char)*p))
p++;
const char *token = p;
while (p < end && !isspace((unsigned char)*p))
p++;
if (p > token)
emit(context, token, (size_t)(p - token));
}
}
/* One line of a value: on the key's line, or a continuation below it. */
static void py_cfg_value(const char *section, const char *key, size_t key_length, const char *text,
const char *end, cbm_py_layout_emit_fn emit, void *context) {
text = py_skip_blank(text, end);
if (!strcmp(section, "options") && key_length == 11 && !memcmp(key, "package_dir", 11)) {
/* `= src` maps the root package; `name = dir` maps one package only. */
if (text < end && *text == '=')
py_cfg_tokens(text + 1, end, emit, context);
return;
}
if ((!strcmp(section, "options.packages.find") && key_length == 5 &&
!memcmp(key, "where", 5)) ||
((!strcmp(section, "tool:pytest") || !strcmp(section, "pytest")) && key_length == 10 &&
!memcmp(key, "pythonpath", 10)))
py_cfg_tokens(text, end, emit, context);
}
static void py_parse_cfg(const char *source, const char *end, cbm_py_layout_emit_fn emit,
void *context) {
char section[PY_LAYOUT_SECTION_MAX] = "";
const char *key = NULL;
size_t key_length = 0;
for (const char *line = source; line < end;) {
const char *line_end = py_line_end(line, end);
const char *next = line_end < end ? line_end + 1 : end;
const char *start = py_skip_blank(line, line_end);
if (start >= line_end || *start == '#' || *start == ';') {
line = next;
continue;
}
if (start == line && py_toml_section(start, line_end, section, sizeof(section))) {
key = NULL;
} else if (start != line) {
if (key)
py_cfg_value(section, key, key_length, start, line_end, emit, context);
} else {
const char *equals = memchr(start, '=', (size_t)(line_end - start));
key = start;
const char *stop = equals ? equals : line_end;
while (stop > key && (stop[-1] == ' ' || stop[-1] == '\t'))
stop--;
key_length = (size_t)(stop - key);
if (equals)
py_cfg_value(section, key, key_length, equals + 1, line_end, emit, context);
}
line = next;
}
}
/* ── setup.py ────────────────────────────────────────────────────────── */
static const char *py_find(const char *p, const char *end, const char *text) {
size_t length = strlen(text);
for (; p && (size_t)(end - p) >= length; p++) {
p = memchr(p, text[0], (size_t)(end - p));
if (!p || (size_t)(end - p) < length)
return NULL;
if (!memcmp(p, text, length))
return p;
}
return NULL;
}
static const char *py_literal(const char *p, const char *end, const char **text, size_t *length) {
p = p < end ? p : end;
while (p < end && isspace((unsigned char)*p))
p++;
if (p >= end || (*p != '"' && *p != '\''))
return NULL;
const char *close = p + 1;
while (close < end && *close != *p && *close != '\n')
close++;
if (close >= end || *close != *p)
return NULL;
*text = p + 1;
*length = (size_t)(close - p - 1);
return close + 1;
}
/* `package_dir={"": "src"}` and `find_packages("src")`, as literals. */
static void py_parse_setup_py(const char *source, const char *end, cbm_py_layout_emit_fn emit,
void *context) {
const char *text = NULL;
size_t length = 0;
for (const char *p = py_find(source, end, "package_dir"); p;
p = py_find(p + 1, end, "package_dir")) {
const char *q = p + strlen("package_dir");
while (q < end && isspace((unsigned char)*q))
q++;
if (q >= end || *q != '=')
continue;
q++;
while (q < end && isspace((unsigned char)*q))
q++;
if (q >= end || *q != '{')
continue;
q = py_literal(q + 1, end, &text, &length);
if (!q || length)
continue;
while (q < end && isspace((unsigned char)*q))
q++;
if (q >= end || *q != ':')
continue;
if (py_literal(q + 1, end, &text, &length))
emit(context, text, length);
}
static const char *const finders[] = {"find_packages(", "find_namespace_packages("};
for (size_t f = 0; f < sizeof(finders) / sizeof(finders[0]); f++) {
for (const char *p = py_find(source, end, finders[f]); p;
p = py_find(p + 1, end, finders[f])) {
const char *q = py_past_spaces(p + strlen(finders[f]), end);
if (end - q > 5 && !memcmp(q, "where", 5)) {
q = py_past_spaces(q + 5, end);
if (q >= end || *q != '=')
continue;
q++;
}
if (py_literal(q, end, &text, &length))
emit(context, text, length);
}
}
}
void cbm_py_layout_parse(const char *basename, const char *source, size_t source_len,
cbm_py_layout_emit_fn emit, void *context) {
const char *end = source + source_len;
if (!strcmp(basename, "pyproject.toml"))
py_parse_toml(source, end, emit, context);
else if (!strcmp(basename, "setup.cfg") || !strcmp(basename, "tox.ini") ||
!strcmp(basename, "pytest.ini"))
py_parse_cfg(source, end, emit, context);
else if (!strcmp(basename, "setup.py"))
py_parse_setup_py(source, end, emit, context);
}
void cbm_py_layout_workspace_members(const char *source, size_t source_len,
cbm_py_layout_emit_fn emit, void *context) {
const char *end = source + source_len;
char section[PY_LAYOUT_SECTION_MAX] = "";
for (const char *line = source; line < end;) {
const char *start = py_skip_blank(line, end);
const char *line_end = py_line_end(start, end);
const char *next = line_end < end ? line_end + 1 : end;
if (py_toml_section(start, line_end, section, sizeof(section))) {
line = next;
continue;
}
if (!strcmp(section, "tool.uv.workspace") && line_end - start > 7 &&
!memcmp(start, "members", 7)) {
const char *value = py_skip_blank(start + 7, line_end);
if (value < line_end && *value == '=') {
value = py_skip_blank(value + 1, line_end);
const char *value_end = py_toml_value_end(value, end);
py_toml_strings(value, value_end, PY_FORM_STRINGS, emit, context);
return;
}
}
line = next;
}
}
bool cbm_py_layout_member_matches(const char *pattern, const char *directory) {
size_t length = strlen(pattern);
if (length < 2 || strcmp(pattern + length - 2, "/*"))
return !strcmp(pattern, directory);
return !strncmp(pattern, directory, length - 1) && directory[length - 1] &&
!strchr(directory + length - 1, '/');
}
/* ── Standard library ────────────────────────────────────────────────── */
/* Sorted. CPython's sys.stdlib_module_names, top-level public modules. */
static const char *const py_stdlib_modules[] = {
"abc",
"aifc",
"antigravity",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"configparser",
"contextlib",
"contextvars",
"copy",
"copyreg",
"crypt",
"csv",
"ctypes",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"doctest",
"email",
"encodings",
"ensurepip",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"fractions",
"ftplib",
"functools",
"gc",
"genericpath",
"getopt",
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"idlelib",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"netrc",
"nis",
"nntplib",
"nt",
"ntpath",
"nturl2path",
"numbers",
"opcode",
"operator",
"optparse",
"os",
"ossaudiodev",
"pathlib",
"pdb",
"pickle",
"pickletools",
"pipes",
"pkgutil",
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"pydoc_data",
"pyexpat",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
"shutil",
"signal",
"site",
"smtpd",
"smtplib",
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre_compile",
"sre_constants",
"sre_parse",
"ssl",
"stat",
"statistics",
"string",
"stringprep",
"struct",
"subprocess",
"sunau",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"textwrap",
"this",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"tomllib",
"trace",
"traceback",
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xmlrpc",
"zipapp",
"zipfile",
"zipimport",
"zlib",
"zoneinfo",
};
bool cbm_py_layout_is_stdlib(const char *name, size_t length) {
size_t low = 0, high = sizeof(py_stdlib_modules) / sizeof(py_stdlib_modules[0]);
while (low < high) {
size_t middle = low + (high - low) / 2;
int order = strncmp(py_stdlib_modules[middle], name, length);
if (!order && py_stdlib_modules[middle][length])
order = 1;
if (!order)
return true;
if (order < 0)
low = middle + 1;
else
high = middle;
}
return false;
}
+36
View File
@@ -0,0 +1,36 @@
#ifndef CBM_PIPELINE_PY_LAYOUT_H
#define CBM_PIPELINE_PY_LAYOUT_H
#include <stdbool.h>
#include <stddef.h>
/* Where a Python project keeps its importable code, as its manifests state it.
* A file under a stated root is the installed module of that path: src/flask/app.py
* is flask.app, and `import flask` anywhere in the project names it. Manifests are read
* as text and nothing in them is run. */
/* pyproject.toml, setup.cfg, setup.py, tox.ini or pytest.ini. */
bool cbm_py_layout_is_manifest(const char *basename);
/* The manifests that can sit in one directory, for a lookup per directory. */
const char *cbm_py_layout_manifest_at(size_t index);
/* Each root the manifest states, as written, relative to its directory. */
typedef void (*cbm_py_layout_emit_fn)(void *context, const char *root, size_t length);
void cbm_py_layout_parse(const char *basename, const char *source, size_t source_len,
cbm_py_layout_emit_fn emit, void *context);
/* Each member pattern of a uv workspace this pyproject.toml declares, as written: its members
* are installed together, so each one's modules are importable from the others. */
void cbm_py_layout_workspace_members(const char *source, size_t source_len,
cbm_py_layout_emit_fn emit, void *context);
/* Whether a project directory matches a member pattern, both relative to the workspace:
* an exact path, or a final `*` standing for one directory name. */
bool cbm_py_layout_member_matches(const char *pattern, const char *directory);
/* A top-level standard library module. The standard library precedes an installed
* distribution on sys.path, so such a name never reaches a source root. */
bool cbm_py_layout_is_stdlib(const char *name, size_t length);
#endif
+4
View File
@@ -1803,7 +1803,9 @@ extern void suite_resolver_rust(void);
extern void suite_resolver_py_types(void);
extern void suite_resolver_python(void);
extern void suite_resolver_python_members(void);
extern void suite_resolver_python_layout(void);
extern void suite_resolver_python_flow(void);
extern void suite_resolver_python_shapes(void);
extern void suite_resolver_java(void);
extern void suite_resolver_oo(void);
extern void suite_resolver_diagnostics(void);
@@ -2446,7 +2448,9 @@ int main(int argc, char **argv) {
RUN_SELECTED_SUITE(resolver_py_types);
RUN_SELECTED_SUITE(resolver_python);
RUN_SELECTED_SUITE(resolver_python_members);
RUN_SELECTED_SUITE(resolver_python_layout);
RUN_SELECTED_SUITE(resolver_python_flow);
RUN_SELECTED_SUITE(resolver_python_shapes);
RUN_SELECTED_SUITE(resolver_java);
RUN_SELECTED_SUITE(resolver_oo);
RUN_SELECTED_SUITE(resolver_diagnostics);
+8 -6
View File
@@ -1119,13 +1119,13 @@ TEST(resolver_binding_python_formatting_keeps_module_escape_and_rebinding_guards
cbm_resolve_result_t proof = cbm_resolve_binding_local(
index, "target", start, start + strlen("target()"), CBM_SOURCE_ORIGIN_RAW);
bool admitted = cbm_resolve_admit(&proof);
bool escaped = cbm_resolve_binding_python_import_escaped(index, "library");
const char *member = NULL;
bool written = cbm_resolve_binding_python_import_write(index, "library", 0, &member);
cbm_arena_destroy(&arena);
cbm_free_result(file);
/* The escape is recorded against library; target itself stays proven. */
/* Formatting a module reads it, and target itself stays proven. */
ASSERT_EQ(admitted, i == 0);
if (i == 0)
ASSERT(escaped);
ASSERT(!written);
}
PASS();
}
@@ -1999,7 +1999,8 @@ static bool python_physical_package_golden(const char *manifest, bool source_lay
const char *library_path = source_layout ? "src/sample/utils.py" : "sample/utils.py";
const char *decoy_path = source_layout ? "decoy/utils.py" : "src/sample/utils.py";
const char *initializer = source_layout ? "src/sample/__init__.py" : "sample/__init__.py";
bool supported = !source_layout || explicit_src;
/* A manifest beside src/ states the layout; its [project].name says nothing. */
bool supported = !source_layout || explicit_src || manifest;
char main[256];
snprintf(main, sizeof(main),
"from %ssample.utils import target\ndef caller():\n return target()\n"
@@ -2029,7 +2030,7 @@ static bool python_physical_package_golden(const char *manifest, bool source_lay
.declaration_text =
"def target(value: Optional[str] = None) -> Tuple[str, str]:"}}};
const RQ_Forbidden forbidden = {
.case_id = "distribution-name-does-not-authorize-src-layout",
.case_id = "no-manifest-states-a-src-layout",
.site = {.file = "main.py", .caller = "caller", .needle = "target()"},
.relation = "CALLS",
.any_target = true};
@@ -2098,6 +2099,7 @@ TEST(resolver_binding_python_distribution_name_is_not_import_root) {
static const char other_distribution[] = "[project]\nname = 'unrelated-dist'\n";
for (int driver = 0; driver < 2; driver++) {
ASSERT(python_physical_package_golden(NULL, false, false, driver));
ASSERT(python_physical_package_golden(NULL, true, false, driver));
ASSERT(python_physical_package_golden(manifest, false, false, driver));
ASSERT(python_physical_package_golden(other_distribution, false, false, driver));
ASSERT(python_physical_package_golden(manifest, true, false, driver));
File diff suppressed because it is too large. Load diff