diff --git a/internal/cbm/resolve/binding.c b/internal/cbm/resolve/binding.c index 9b6f279a2..ccf8e7735 100644 --- a/internal/cbm/resolve/binding.c +++ b/internal/cbm/resolve/binding.c @@ -217,6 +217,12 @@ typedef struct { const char *alias, *of; } python_alias_t; +/* `__go(locals())` at module level: the callee, and where the call starts. */ +typedef struct { + const char *callee; + uint32_t at; +} python_handoff_t; + struct cbm_resolve_binding_index { CBMArena *arena; binding_observation_t *observation; @@ -247,9 +253,11 @@ struct cbm_resolve_binding_index { size_t python_alias_count, python_alias_capacity; size_t python_write_count, python_write_capacity; bool python_effects_complete; - bool python_writes_unseen; /* the audit stopped before the end of the file */ - bool python_name_handed; /* __name__ goes to a call: that may build the module */ - bool python_namespace_open; /* a computed globals() store can add any name */ + bool python_writes_unseen; /* the audit stopped before the end of the file */ + bool python_name_handed; /* __name__ goes to a call: that may build the module */ + bool python_namespace_open; /* a computed globals() store can add any name */ + python_handoff_t *python_handoffs; /* checked once every binder is known */ + size_t python_handoff_count, python_handoff_capacity; python_narrow_t *python_narrows; /* sorted by from, outermost first, once the walk ends */ size_t python_narrow_count, python_narrow_capacity; python_store_t *python_stores; /* attribute stores, sorted by chain then position */ @@ -1025,6 +1033,125 @@ static bool python_name_shadowed(cbm_resolve_binding_index_t *index, const char return true; } +/* `eval('__IPYTHON__')`, pydantic's in_ipython: code that is one literal name only reads it. + * The argument list holds that string alone, with no escapes, interpolation or prefix. */ +static bool python_code_reads_one_name(cbm_resolve_binding_index_t *index, TSNode args) { + if (ts_node_is_null(args) || ts_node_named_child_count(args) != 1) + return false; + TSNode code = ts_node_named_child(args, 0); + if (!kind_is(code, "string") || ts_node_named_child_count(code) != 3) + return false; + TSNode start = ts_node_named_child(code, 0), content = ts_node_named_child(code, 1); + if (!kind_is(start, "string_start") || !kind_is(content, "string_content") || + ts_node_named_child_count(content) != 0 || + ts_node_end_byte(start) - ts_node_start_byte(start) != 1) + return false; + uint32_t from = ts_node_start_byte(content), to = ts_node_end_byte(content); + if (to <= from || to > index->source_len || + (index->source[from] >= '0' && index->source[from] <= '9')) + return false; + for (uint32_t at = from; at < to; at++) { + char c = index->source[at]; + if (!(c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9'))) + return false; + } + return true; +} + +static bool python_spells(const cbm_resolve_binding_index_t *index, TSNode node, const char *word); + +/* Names through which code can reach a namespace it was handed by another name. */ +static const char *const python_namespace_reach[] = {"locals", "vars", "eval", + "exec", "f_locals", "_getframe", + "currentframe", "getargvalues", NULL}; + +/* One use of a namespace alias only reads: a membership test, an item read, or a read-only + * method. An item that is an assignment, del, for or with target writes. */ +static bool python_alias_use_reads(cbm_resolve_binding_index_t *index, TSNode use) { + TSNode parent = ts_node_parent(use); + if (kind_is(parent, "comparison_operator")) + return true; + if (kind_is(parent, "attribute") && ts_node_eq(field(parent, "object"), use)) { + const char *member = node_text(index, field(parent, "attribute")); + TSNode call = ts_node_parent(parent); + return member && kind_is(call, "call") && ts_node_eq(field(call, "function"), parent) && + (!strcmp(member, "get") || !strcmp(member, "keys") || !strcmp(member, "items") || + !strcmp(member, "values") || !strcmp(member, "copy")); + } + if (!kind_is(parent, "subscript") || !ts_node_eq(field(parent, "value"), use)) + return false; + TSNode target = parent, user = ts_node_parent(parent); + static const char *const wrappers[] = {"pattern_list", + "tuple_pattern", + "list_pattern", + "expression_list", + "parenthesized_expression", + "tuple", + "list", + "list_splat_pattern", + NULL}; + for (unsigned depth = 0; depth < BINDING_MAX_DEPTH && listed(ts_node_type(user), wrappers); + depth++) { + target = user; + user = ts_node_parent(user); + } + if (kind_is(user, "delete_statement") || kind_is(user, "as_pattern_target")) + return false; + return !((kind_is(user, "assignment") || kind_is(user, "augmented_assignment") || + kind_is(user, "for_statement") || kind_is(user, "for_in_clause")) && + ts_node_eq(field(user, "left"), target)); +} + +/* `glob = globals()` in a function, as sqlalchemy's exc.py and automap.py do. The alias writes + * nothing if every other use of it in the function only reads and nothing there can reach it + * by another name. A module-level alias is a module attribute another file can write through, + * so it stays an effect. One cursor pass over the function per alias. */ +static bool python_view_alias_reads(cbm_resolve_binding_index_t *index, TSNode call) { + TSNode assignment = ts_node_parent(call), left = field(assignment, "left"); + if (!kind_is(assignment, "assignment") || !ts_node_eq(field(assignment, "right"), call) || + !kind_is(left, "identifier") || + !kind_is(ts_node_parent(assignment), "expression_statement")) + return false; + const char *alias = node_text(index, left); + TSNode scope = ts_node_parent(assignment); + for (unsigned depth = 0; depth < BINDING_MAX_DEPTH && !ts_node_is_null(scope) && + !kind_is(scope, "function_definition") && !kind_is(scope, "module"); + depth++) { + if (kind_is(scope, "class_definition") || kind_is(scope, "lambda")) + return false; + scope = ts_node_parent(scope); + } + if (!alias || !kind_is(scope, "function_definition")) + return false; + TSNode own = field(call, "function"); + bool reads = true; + TSTreeCursor cursor = ts_tree_cursor_new(scope); + bool more = ts_tree_cursor_goto_first_child(&cursor); + unsigned depth = 1; + while (more && reads) { + TSNode node = ts_tree_cursor_current_node(&cursor); + if (kind_is(node, "identifier") && !ts_node_eq(node, own)) { + for (size_t i = 0; reads && python_namespace_reach[i]; i++) + reads = !python_spells(index, node, python_namespace_reach[i]); + if (reads && !ts_node_eq(node, left) && python_spells(index, node, alias)) + reads = python_alias_use_reads(index, node); + } + if (ts_tree_cursor_goto_first_child(&cursor)) { + depth++; + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (--depth == 0 || !ts_tree_cursor_goto_parent(&cursor)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + return reads; +} + /* exec and eval write the namespace they run in, which an explicit globals * argument replaces. globals() is a write only if it can be written through: a * membership test, an item read, a read-only method or a splat into a new dict @@ -1040,7 +1167,7 @@ static bool python_namespace_call_writes(cbm_resolve_binding_index_t *index, TSN for (uint32_t i = 0; !ts_node_is_null(args) && i < ts_node_named_child_count(args); i++) positional += !kind_is(ts_node_named_child(args, i), "keyword_argument") && !kind_is(ts_node_named_child(args, i), "comment"); - return positional < 2; + return positional < 2 && !python_code_reads_one_name(index, args); } TSNode parent = ts_node_parent(call); if (kind_is(parent, "comparison_operator") || kind_is(parent, "dictionary_splat")) @@ -1072,6 +1199,29 @@ static bool python_namespace_call_writes(cbm_resolve_binding_index_t *index, TSN (strcmp(member, "get") && strcmp(member, "keys") && strcmp(member, "items") && strcmp(member, "values") && strcmp(member, "copy")); } + return !python_view_alias_reads(index, call); +} + +/* `__go(locals())` at module level hands the namespace to the callee's first parameter, as + * sqlalchemy's package inits do. The callee is known only once every binder is seen, so the + * call is noted here and python_settle_handoffs decides it. */ +static bool python_note_handoff(cbm_resolve_binding_index_t *index, TSNode view) { + TSNode arguments = ts_node_parent(view); + TSNode call = ts_node_parent(arguments); + if (index->current_scope != 0 || !kind_is(arguments, "argument_list") || + !kind_is(call, "call") || !ts_node_eq(ts_node_named_child(arguments, 0), view)) + return false; + TSNode callee = field(call, "function"); + const char *name = kind_is(callee, "identifier") ? node_text(index, callee) : NULL; + python_handoff_t *rows = + name ? grow_array(index, index->python_handoffs, index->python_handoff_count, + &index->python_handoff_capacity, sizeof(*rows)) + : NULL; + if (!rows) + return false; + index->python_handoffs = rows; + rows[index->python_handoff_count++] = + (python_handoff_t){.callee = name, .at = ts_node_start_byte(call)}; return true; } @@ -1098,7 +1248,10 @@ static bool python_namespace_view(cbm_resolve_binding_index_t *index, TSNode nod bool in_function = index->current_scope && index->current_scope < index->scope_count && !index->scopes[index->current_scope].class_scope; TSNode args = field(call, "arguments"); - return !in_function && (ts_node_is_null(args) || ts_node_named_child_count(args) == 0); + if (in_function || (!ts_node_is_null(args) && ts_node_named_child_count(args) != 0) || + python_name_shadowed(index, name)) + return false; + return !python_note_handoff(index, call); } /* A name bound by `if TYPE_CHECKING:` exists only for annotations; the @@ -5857,6 +6010,72 @@ static bool javascript_module_by_source(const CBMFileResult *result, CBMLanguage (!strcmp(extension, ".mjs") || !strcmp(extension, ".cjs") || !strcmp(extension, ".jsx")); } +/* The callee's body never spells the parameter that receives the namespace, nor a way to + * reach it by another name: locals, vars, eval, exec, or a frame's locals. One cursor pass. */ +static bool python_handoff_body_inert(cbm_resolve_binding_index_t *index, TSNode function) { + const char *parameter = python_first_parameter(index, function); + TSNode body = field(function, "body"); + if (!parameter || ts_node_is_null(body)) + return false; + bool mentioned = false; + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool more = ts_tree_cursor_goto_first_child(&cursor); + unsigned depth = 1; + while (more && !mentioned) { + TSNode node = ts_tree_cursor_current_node(&cursor); + if (kind_is(node, "identifier")) { + mentioned = python_spells(index, node, parameter); + for (size_t i = 0; !mentioned && python_namespace_reach[i]; i++) + mentioned = python_spells(index, node, python_namespace_reach[i]); + } + if (ts_tree_cursor_goto_first_child(&cursor)) { + depth++; + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (--depth == 0 || !ts_tree_cursor_goto_parent(&cursor)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + return !mentioned; +} + +/* A hand-off writes nothing when the callee is this module's one undecorated def, bound before + * the call, whose body cannot reach the namespace it was given. What the body does by other + * means (global, imports, writes through a module) the walk has already seen. Anything else + * stays a whole-namespace effect. One lookup per hand-off; a body is scanned once per def. */ +static void python_settle_handoffs(cbm_resolve_binding_index_t *index, TSNode root) { + uint32_t verified = UINT32_MAX; + for (size_t i = 0; + i < index->python_handoff_count && index->analysis == CBM_RESOLVE_ANALYSIS_COMPLETE; i++) { + const python_handoff_t *handoff = &index->python_handoffs[i]; + bool present = false; + const binding_name_t *row = find_scoped_name( + index, handoff->callee, strlen(handoff->callee), 0, &present, JS_SPACE_ANY); + bool inert = row && row->at < handoff->at && (row->flags & BINDING_DECLARATION) && + !(row->flags & + (BINDING_BLOCKED | BINDING_DECORATED | BINDING_WRITTEN | BINDING_IMPORT)); + if (inert && row->at != verified) { + TSNode function = ts_node_descendant_for_byte_range(root, row->at, row->at); + for (unsigned depth = 0; depth < BINDING_MAX_DEPTH && !ts_node_is_null(function) && + !(kind_is(function, "function_definition") && + ts_node_start_byte(function) == row->at); + depth++) + function = ts_node_parent(function); + inert = kind_is(function, "function_definition") && + ts_node_start_byte(function) == row->at && + python_handoff_body_inert(index, function); + if (inert) + verified = row->at; + } + if (!inert) + fail_index(index, CBM_RESOLVE_ANALYSIS_UNSUPPORTED, "dynamic_namespace_effect"); + } +} + static cbm_resolve_binding_index_t *binding_build(CBMArena *arena, const CBMFileResult *result, CBMLanguage language, const char *source, size_t source_len, bool declared_module, @@ -6014,6 +6233,8 @@ static cbm_resolve_binding_index_t *binding_build(CBMArena *arena, const CBMFile if (language == CBM_LANG_PYTHON) python_refuse_class_comprehension_reads(index); } + if (language == CBM_LANG_PYTHON && tree && index->python_handoff_count) + python_settle_handoffs(index, ts_tree_root_node(tree)); if (javascript_language(language)) { js_note_markup_writes(index); js_apply_writes(index); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 95d7a708e..5652d6c32 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -24004,12 +24004,8 @@ static const RQ_Expected py_reexport_expected[] = { .label = "Class", .declaration_line = 1, .declaration_text = "class Thing:"}, - .mode = RQ_KNOWN_GAP, - .gap_id = "python-reexport.go_reexport", - .why = "Package initialization through __go(locals()) lacks a closed interprocedural " - "namespace-effect proof.", - .tried = "The helper passes Thing to the fixture prepare function, which returns it " - "unchanged; generic locals and calls remain proof barriers."}, + /* __go never spells lcls: handing it the namespace writes nothing. */ + .mode = RQ_REQUIRED}, {.case_id = "python-reexport.excl_barrier", .site = {.file = "app/main.py", .caller = "app.main.excl_barrier", @@ -24045,13 +24041,9 @@ static const RQ_Expected py_reexport_expected[] = { .label = "Function", .declaration_line = 5, .declaration_text = "def base():"}, - .mode = RQ_KNOWN_GAP, - .gap_id = "python-reexport.excl_go_submodule", - .why = "The helper import requires existing-package-attribute precedence and a closed " - "namespace-effect proof.", - .tried = - "from . import base first finds the existing function attribute imported from .impl, so " - "base.py is not loaded by that statement; the generic helper-effect barrier remains."}, + /* from . import base inside __go finds the function attribute imported from .impl, so + * base.py is not loaded and gop.base stays that function. */ + .mode = RQ_REQUIRED}, {.case_id = "python-reexport.compat_runtime", .site = {.file = "app/main.py", .caller = "app.main.excl_compat", diff --git a/tests/test_resolver_binding_generic.c b/tests/test_resolver_binding_generic.c index 913ee9687..63a566dcf 100644 --- a/tests/test_resolver_binding_generic.c +++ b/tests/test_resolver_binding_generic.c @@ -1407,7 +1407,10 @@ TEST(resolver_binding_python_annotation_scope_and_effects) { {"callback: object = other\ndef caller():\n callback()\n", false, false, true}, /* An annotation expression is a read, as on an assignment: the import stands. */ {"callback: factory()\ndef caller():\n callback()\n", true, true, true}, - {"callback: eval('object')\ndef caller():\n callback()\n", false, false, false}, + /* eval of one literal name only reads it; eval of other code can rebind callback. */ + {"callback: eval('object')\ndef caller():\n callback()\n", true, true, true}, + {"callback: eval(code)\ndef caller():\n callback()\n", false, false, false}, + {"callback: eval('(callback := 1)')\ndef caller():\n callback()\n", false, false, false}, {"__annotations__ = box\ncallback: object\ndef caller():\n callback()\n", false, false, true}, }; diff --git a/tests/test_resolver_python.c b/tests/test_resolver_python.c index 92101e8d7..6a590a344 100644 --- a/tests/test_resolver_python.c +++ b/tests/test_resolver_python.c @@ -306,7 +306,14 @@ TEST(resolver_python_only_real_namespace_effects_are_dynamic) { "def known(code):\n return eval(code, {}, {'x': 1})\n", "import sys\ndef seen():\n return hasattr(sys.modules[__name__], 'apps')\n", "def where():\n return '__file__' in globals()\n", - "def merged():\n return {**globals(), 'x': 1}\n"}; + "def merged():\n return {**globals(), 'x': 1}\n", + /* pydantic/v1/utils.py in_ipython: one literal name is only read */ + "def in_ipython():\n try:\n eval('__IPYTHON__')\n except NameError:\n" + " return False\n return True\n", + "seen = eval(\"local\")\n", + /* lib/sqlalchemy/exc.py: an alias of the namespace that is only read */ + "def translate(name):\n glob = globals()\n if name in glob and callable(glob[name]):\n" + " return glob[name]\n return glob.get(name)\n"}; for (size_t i = 0; i < sizeof(inert) / sizeof(inert[0]); i++) { char source[512]; snprintf(source, sizeof(source), @@ -315,6 +322,26 @@ TEST(resolver_python_only_real_namespace_effects_are_dynamic) { ASSERT(python_expect(inert[i], &file, 1, "CALLS", "main.py", "local", "Function", 1)); } static const char *effects[] = {"exec('local = None')\n", "eval('1')\n", + "eval('(local := None)')\n", "eval(name)\n", + "eval('local' + suffix)\n", "eval(u'local = None')\n", + "glob = globals()\nglob['local'] = None\n", + "__locals = locals()\n__locals['local'] = None\n", + /* httpx/__init__.py: read here, but a module attribute + * another file can write through */ + "__locals = locals()\nfor __name in []:\n" + " setattr(__locals[__name], '__module__', 'x')\n", + "def f():\n glob = globals()\n glob.update(local=None)\n", + "def f():\n glob = globals()\n del glob['local']\n", + "def f():\n glob = globals()\n keep(glob)\n", + "def f():\n glob = globals()\n return glob\n", + "def f():\n glob = globals()\n" + " for glob['local'] in [None]:\n pass\n", + "def f():\n glob = globals()\n a, glob['local'] = 1, None\n", + "def f():\n glob = globals()\n glob['local'] += 1\n", + "def f():\n glob = globals()\n" + " locals()['glob']['local'] = None\n", + "def f():\n glob = globals()\n" + " def g():\n glob['local'] = None\n g()\n", "globals()['local'] = None\n", "locals()['local'] = None\n", "vars()['local'] = None\n", "setattr(sys.modules[__name__], 'local', None)\n"}; @@ -1954,6 +1981,72 @@ TEST(resolver_python_package_importing_its_own_child) { PASS(); } +/* sqlalchemy's package inits end with `__go(locals())`. Handing the namespace to the module's + * own def writes nothing when that def never spells the parameter, nor reaches it another way; + * what the body does by other means is seen as anywhere else. */ +TEST(resolver_python_package_init_hands_its_namespace_on) { + static const char make[] = "def make():\n return 1\n"; + static const char user[] = "from pkg import make\ndef run():\n return make()\n"; + static const char direct[] = "import pkg\ndef run():\n return pkg.make()\n"; + static const struct { + const char *tail; + int expected; + } cases[] = { + /* lib/sqlalchemy/orm/__init__.py */ + {"from typing import Any\nfrom . import other as _sa_util\n\n\n" + "def __go(lcls: Any) -> None:\n _sa_util.preloaded.import_prefix(\"pkg\")\n\n\n" + "__go(locals())\n", + 1}, + /* lib/sqlalchemy/__init__.py */ + {"def __go(lcls):\n from . import other\n\n other._token = \"\".join([])\n\n\n" + "__go(locals())\n", + 1}, + /* lib/sqlalchemy/orm/collections.py: globals the body rebinds are its own names */ + {"def __go(lcls):\n global keyfunc\n from .other import keyfunc\n\n\n" + "__go(locals())\n", + 1}, + {"def __go(lcls):\n pass\n\n\nresult = __go(vars())\n", 1}, + /* The body reaches the namespace it was given. */ + {"def __go(lcls):\n lcls['make'] = None\n\n\n__go(locals())\n", 0}, + {"def __go(lcls):\n helper(lcls)\n\n\ndef helper(d):\n pass\n\n\n__go(locals())\n", + 0}, + {"def __go(lcls):\n locals()['lcls']['make'] = None\n\n\n__go(locals())\n", 0}, + {"def __go(lcls):\n eval(\"lcls.clear()\")\n\n\n__go(locals())\n", 0}, + {"import sys\n\n\ndef __go(lcls):\n sys._getframe().f_locals['lcls'].clear()\n\n\n" + "__go(locals())\n", + 0}, + {"def __go(lcls):\n def inner():\n lcls.clear()\n inner()\n\n\n" + "__go(locals())\n", + 0}, + {"def __go(first, lcls=None):\n first['make'] = None\n\n\n__go(locals())\n", 0}, + {"def __go(*args):\n args[0]['make'] = None\n\n\n__go(locals())\n", 0}, + {"def __go(lcls):\n global make\n make = None\n\n\n__go(locals())\n", 0}, + /* The callee is not provably this module's one plain def at the call. */ + {"from .other import __go\n__go(locals())\n", 0}, + {"def __go(lcls):\n pass\n\n\n__go = print\n__go(locals())\n", 0}, + {"__go(locals())\n\n\ndef __go(lcls):\n pass\n", 0}, + {"def wrap(f):\n return f\n\n\n@wrap\ndef __go(lcls):\n pass\n\n\n__go(locals())\n", + 0}, + {"def __go(lcls):\n pass\n\n\nfrom .other import *\n__go(locals())\n", 0}, + {"def __go(other, lcls):\n pass\n\n\n__go(1, locals())\n", 0}, + {"__go = lambda lcls: None\n__go(locals())\n", 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char init[640]; + snprintf(init, sizeof(init), "from .impl import make\n%s", cases[i].tail); + const python_proof_file_t files[] = {{"pkg/__init__.py", init}, + {"pkg/impl.py", make}, + {"pkg/other.py", "keyfunc = 1\n__all__ = []\n"}, + {"user.py", user}, + {"main.py", direct}}; + ASSERT(python_expect_from(cases[i].tail, files, 5, "user.py", "CALLS", "pkg/impl.py", + "make", "Function", cases[i].expected)); + ASSERT(python_expect_from(cases[i].tail, files, 5, "main.py", "CALLS", "pkg/impl.py", + "make", "Function", cases[i].expected)); + } + PASS(); +} + /* ── What a call of a class, and of an unannotated method, gives ────────── * Forms from django and celery. */ TEST(resolver_python_special_methods_that_read_through_the_base) { @@ -2811,6 +2904,7 @@ SUITE(resolver_python_layout) { RUN_TEST(resolver_python_names_a_module_states_only_for_checkers); RUN_TEST(resolver_python_overload_stubs_are_one_declaration); RUN_TEST(resolver_python_loading_a_child_the_parent_imported); + RUN_TEST(resolver_python_package_init_hands_its_namespace_on); python_route_release(&python_routes[0]); python_route_release(&python_routes[1]); }