From 50ee111714baa9d3bfe9bdba3b81e27cd1bdfe38 Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Fri, 28 Aug 2026 17:26:26 +0200 Subject: [PATCH 01/19] fix(pipeline): suppress weak short-name matches for Go selector calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Go selector call x.foo() whose receiver the Go LSP cannot type falls through to the generic registry resolver, which binds it by bare short name to an arbitrary same-named project symbol. Stdlib calls are the worst case: f.Close() on an *os.File gets a CALLS edge to whatever project Close wins candidate ranking (measured on a real Go repo: confidence 0.11, 15 candidates; suffix_match + unique_name were 36% of all CALLS edges, and one 14-line stdlib-only function got 3 out of 3 false outbound edges). Extend the TS/JS receiver-aware guard (#592/#606) to Go: - extract_calls.c: flag Go call_expression with a selector_expression callee as is_method, mirroring the TS/JS member_expression flag. - registry.c: add cbm_go_suppress_weak_method_match. Unlike the TS/JS drop-list, field_type_hint is KEPT (Go struct fields carry declared types, so the hint is receiver-aware — lrp_go_s8_field_type_hint), and unique_name is dropped only when its confidence carries the import-unreachability penalty (the stdlib-hijack shape); an unpenalized lone candidate inside the caller's import closure never enters the field-type-hint upgrade and must survive. - pass_calls.c / pass_parallel.c: feed the Go gate next to the TS/JS one; the drop still defers to the emit path so service/route/HTTP edges stay main-identical. Reproduce-first: pipeline_go_receiver_suppresses_weak_method_edge is RED without the extractor flag (the f.Close -> project Close edge exists) and GREEN with it; typed same-package calls, bare local calls and import-qualified cross-package calls still resolve. The old extraction contract test used Go as the flag-exempt language — Python takes that role, and extract_go_selector_call_flags_is_method pins the new behavior. Signed-off-by: Ilya Brykau --- internal/cbm/extract_calls.c | 18 +++++++ src/pipeline/pass_calls.c | 12 ++++- src/pipeline/pass_parallel.c | 8 ++- src/pipeline/pipeline.h | 9 ++++ src/pipeline/registry.c | 27 +++++++++++ tests/test_extraction.c | 39 +++++++++++++-- tests/test_pipeline.c | 94 ++++++++++++++++++++++++++++++++++++ tests/test_registry.c | 42 ++++++++++++++++ 8 files changed, 242 insertions(+), 7 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 47250203d..6d315faf0 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -3712,6 +3712,24 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML } } } + // Go receiver-aware guard (same direction as the TS/JS flag above). + // Flag a selector call x.foo(). The Go AST cannot separate a method + // call on a value from a package-qualified call — but every selector + // call the Go LSP or the import/qualified registry strategies CAN + // place never reaches the weak short-name guards, so the flag only + // bites on unresolvable receivers (`f.Close()` on an os.File, + // `sha256.New()` behind an unindexed import), where a project-wide + // short-name match fabricates an edge to an unrelated project + // symbol sharing the name. Bare calls (helper()) keep + // is_method=false and resolve same-module/import paths as before. + if (ctx->language == CBM_LANG_GO && + strcmp(ts_node_type(node), "call_expression") == 0) { + TSNode gofn = ts_node_child_by_field_name(node, TS_FIELD("function")); + if (!ts_node_is_null(gofn) && + strcmp(ts_node_type(gofn), "selector_expression") == 0) { + call.is_method = true; + } + } TSNode args = ts_node_child_by_field_name(node, TS_FIELD("arguments")); /* tree-sitter-elixir attaches NO field name to a call's arguments diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index c9744dda2..2d96d440d 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -623,7 +623,13 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, * language gated on only one resolver produces an edge on the sequential * path and not the parallel one (or vice versa), breaking MT determinism. * ArkTS belongs to the JS/TS family here (#1842); dropping it would - * reintroduce the #592/#606 false-edge class for .ets files. */ + * reintroduce the #592/#606 false-edge class for .ets files. + * + * Go (#1906) rides the same deferred-drop plumbing through its OWN + * predicate: its drop-list differs (field_type_hint is receiver-aware for + * Go, and unique_name drops only when import-unreachability-penalized), so + * it composes via cbm_go_suppress_weak_method_match instead of widening + * the shared gate. Same lockstep rule: mirror pass_parallel.c. */ bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; @@ -636,7 +642,9 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, bool drop_plain_call = cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, - call->callee_is_locally_bound, res.strategy); + call->callee_is_locally_bound, res.strategy) || + cbm_go_suppress_weak_method_match(lang == CBM_LANG_GO, call->is_method, res.strategy, + res.confidence); /* Service-pattern HTTP/ASYNC calls to an EXTERNAL client library (e.g. * `requests.get("/api/orders/{id}")`) resolve to a QN containing the library diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index f5e4ab4d2..3e896c1f4 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2834,7 +2834,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * #606 direction. * * This language set MUST match the one in pass_calls.c exactly — see the - * note there. ArkTS belongs to the JS/TS family (#1842). */ + * note there. ArkTS belongs to the JS/TS family (#1842). Go (#1906) + * composes via its own predicate (different drop-list — see + * cbm_go_suppress_weak_method_match), mirrored in pass_calls.c. */ bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; @@ -2844,7 +2846,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB bool drop_plain_call = cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, - call->callee_is_locally_bound, res.strategy); + call->callee_is_locally_bound, res.strategy) || + cbm_go_suppress_weak_method_match(lang == CBM_LANG_GO, call->is_method, res.strategy, + res.confidence); /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the * service signal lives in the callee_name. The registry can mis-resolve diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 954686849..764f8bd3c 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -297,6 +297,15 @@ bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *st bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_bound, const char *strategy); +/* Go analog of the TS/JS guard, same failure class: a selector call whose + * receiver the Go LSP could not type must not be bound by a receiver-blind + * short-name strategy. Drops suffix_match / fuzzy always, and unique_name only + * when its confidence is import-unreachability-penalized (the stdlib/vendor + * hijack shape). field_type_hint is deliberately NOT dropped for Go — struct + * fields carry declared types, so the hint is receiver-aware there. */ +bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *strategy, + double confidence); + /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 * and is left alone; same_module / import_map / lsp_* are kept. JS/TS/TSX diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index fa926d3c5..6eabdf227 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -501,6 +501,33 @@ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_b return weak_short_name_strategy(strategy); } +bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *strategy, + double confidence) { + if (!is_go || !is_method || !strategy || !strategy[0]) { + return false; + } + /* Go analog of the TS/JS guard above, same failure class: a selector call + * whose receiver the Go LSP could not type reaches the registry and a bare + * short-name strategy binds it to an arbitrary same-named project symbol + * (`f.Close()` on an os.File -> a project `Close`, suffix_match over 15 + * candidates). Unlike the TS/JS list, field_type_hint is KEPT: a Go struct + * field carries a declared type, so the parallel resolver's field-type + * hint is receiver-aware for Go (lrp_go_s8_field_type_hint), not a + * heuristic. */ + if (strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "fuzzy") == 0) { + return true; + } + /* unique_name is dropped only when PENALIZED: resolve_name_lookup scales + * CONF_UNIQUE_NAME by DEFAULT_CONFIDENCE exactly when the lone candidate + * is not reachable through the caller's imports — the stdlib/vendor + * hijack shape (`io.Copy` -> a project `Copy`). An unpenalized + * unique_name target sits inside the caller's import closure (or the + * file has no imports, e.g. a same-package call) and must be kept — + * dropping it kills genuinely-typed lone-candidate calls that never + * enter the field-type-hint upgrade (candidate_count == 1). */ + return strcmp(strategy, "unique_name") == 0 && confidence < CONF_UNIQUE_NAME; +} + static bool js_ts_family(CBMLanguage lang) { return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 14d75bf8b..4bdab079e 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -5204,9 +5204,14 @@ TEST(extract_perl_method_call_flags_is_method) { /* Languages OUTSIDE the is_method flag set (only Perl and TS/JS/TSX set it) must * be unaffected: a Go method call never sets is_method. */ TEST(extract_flag_exempt_method_call_not_flagged_is_method) { - CBMFileResult *r = extract("package m\n" - "func run(o Obj) { o.Commit(); helper() }\n", - CBM_LANG_GO, "t", "x.go"); + /* Rust is flag-exempt: only Perl, Python, TS/JS and Go set is_method. + * Guards the blast radius of the receiver-aware flags for every other + * language. */ + CBMFileResult *r = extract("fn run(o: Obj) {\n" + " o.commit();\n" + " helper();\n" + "}\n", + CBM_LANG_RUST, "t", "x.rs"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); for (int i = 0; i < r->calls.count; i++) { @@ -5364,6 +5369,33 @@ TEST(extract_python_bare_call_flags_locally_bound_callee) { PASS(); } +TEST(extract_go_selector_call_flags_is_method) { + /* Go selector calls are flagged so the weak-match guard can fire when the + * Go LSP cannot type the receiver; bare calls stay unflagged. */ + CBMFileResult *r = extract("package m\n" + "func run(o Obj) { o.Commit(); helper() }\n", + CBM_LANG_GO, "t", "x.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + bool saw_selector = false; + bool saw_bare = false; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + if (c->callee_name && strstr(c->callee_name, "Commit") != NULL) { + ASSERT_TRUE(c->is_method); + saw_selector = true; + } + if (c->callee_name && strcmp(c->callee_name, "helper") == 0) { + ASSERT_FALSE(c->is_method); + saw_bare = true; + } + } + ASSERT_TRUE(saw_selector); + ASSERT_TRUE(saw_bare); + cbm_free_result(r); + PASS(); +} + /* The bare-call flag is DEPTH-INDEPENDENT, and the binding is UNWOUND when its * scope closes. * @@ -7508,6 +7540,7 @@ SUITE(extraction) { RUN_TEST(extract_python_member_call_flags_is_method); RUN_TEST(extract_python_bare_call_flags_locally_bound_callee); RUN_TEST(extract_python_bare_call_flag_is_depth_independent); + RUN_TEST(extract_go_selector_call_flags_is_method); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 565c72719..dfa5fea20 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5467,6 +5467,99 @@ TEST(pipeline_python_receiver_suppresses_weak_method_edge) { PASS(); } +TEST(pipeline_go_receiver_suppresses_weak_method_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + /* go.mod makes project imports resolvable — real Go repos always have one, + * and import reachability (the unique_name penalty) depends on it. */ + write_temp_file(tmp, "go.mod", "module example.com/myapp\n\ngo 1.22\n"); + /* The lone project symbol named "Close" — a real method. */ + write_temp_file(tmp, "storage/storage.go", + "package storage\n" + "\n" + "type Storage struct{ open bool }\n" + "\n" + "func NewStorage() *Storage { return &Storage{open: true} }\n" + "\n" + "func (s *Storage) Close() {\n" + "\ts.open = false\n" + "}\n" + "\n" + "func Boot() {\n" + "\ts := NewStorage()\n" + "\ts.Close()\n" + "}\n"); + /* Cross-package control target: imported by hash.go, so the caller file + * has a non-empty import map (like any real Go file) and unreachable + * unique_name candidates get the import penalty. */ + write_temp_file(tmp, "util/util.go", + "package util\n" + "\n" + "func Tag() string { return \"t\" }\n"); + /* Stdlib receiver: `f.Close()` closes an *os.File, NOT the project method. + * The Go LSP cannot bind it to a project symbol → the registry would guess + * Close by short name (weak). This is the false edge to suppress — + * the exact shape that attached every file/rows/gzip Close in a real Go + * repo to one unrelated project method. */ + write_temp_file(tmp, "hash/hash.go", + "package hash\n" + "\n" + "import (\n" + "\t\"os\"\n" + "\n" + "\t\"example.com/myapp/util\"\n" + ")\n" + "\n" + "func FileLen(path string) int64 {\n" + "\tf, err := os.Open(path)\n" + "\tif err != nil {\n" + "\t\treturn 0\n" + "\t}\n" + "\tdefer f.Close()\n" + "\tst, err := f.Stat()\n" + "\tif err != nil {\n" + "\t\treturn 0\n" + "\t}\n" + "\treturn st.Size()\n" + "}\n" + "\n" + "func localHelper() int { return 1 }\n" + "\n" + "func CallsLocal() int { return localHelper() }\n" + "\n" + "func UsesUtil() string { return util.Tag() }\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_recv.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* (1) The false edge is suppressed (reproduce-first: RED before the fix). */ + ASSERT_FALSE(cross_file_call_exists(s, project, "FileLen", "Close")); + /* (2) The same-package typed-receiver call survives (LSP / same_module — + * both outside the weak drop-list). */ + ASSERT_TRUE(cross_file_call_exists(s, project, "Boot", "Close")); + /* (3) The bare local call survives (is_method stays false for bare calls). */ + ASSERT_TRUE(cross_file_call_exists(s, project, "CallsLocal", "localHelper")); + /* (4) The import-qualified cross-package call survives (import-aware + * strategies are outside the drop-list). */ + ASSERT_TRUE(cross_file_call_exists(s, project, "UsesUtil", "Tag")); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Fixture for the #1928 cross-language reference-guard probes (sequential and * parallel twins). pad_files > 0 adds filler files to push the index over the * parallel-pipeline threshold, since USAGE/WRITES/READS have one resolver per @@ -14594,6 +14687,7 @@ SUITE(pipeline) { #endif RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); RUN_TEST(pipeline_go_bare_ref_never_binds_field); diff --git a/tests/test_registry.c b/tests/test_registry.c index 247ca61ad..aaeb70abb 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -973,6 +973,46 @@ TEST(weak_call_guards_share_one_drop_list) { PASS(); } +TEST(go_suppress_drops_weak_selector_matches) { + /* Go selector call with an untyped receiver, landed via a receiver-blind + * short-name strategy → drop (same failure class as #592/#606). + * suffix_match/fuzzy drop at any confidence; unique_name drops only when + * import-unreachability-penalized (CONF_UNIQUE_NAME 0.75 * 0.5 = 0.375 — + * the `io.Copy` -> project `Copy` stdlib-hijack shape). */ + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "suffix_match", 0.9)); + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "suffix_match", 0.11)); + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "fuzzy", 0.9)); + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "unique_name", 0.375)); + PASS(); +} + +TEST(go_suppress_keeps_typed_and_import_aware_matches) { + /* Unpenalized unique_name = lone candidate inside the caller's import + * closure (or an import-free file, e.g. same-package) — a genuinely-typed + * lone-candidate call never enters the field-type-hint upgrade, so it must + * survive (lrp_go_s8_field_type_hint). */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "unique_name", 0.75)); + /* field_type_hint is receiver-aware for Go — struct fields carry declared + * types (lrp_go_s8_field_type_hint) — so it stays, unlike the TS/JS list. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "field_type_hint", 0.85)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "same_module", 0.9)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "import_map", 0.95)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "import_map_suffix", 0.9)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "qualified_suffix", 0.9)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "callee_suffix", 0.5)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "service_pattern", 0.5)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "lsp_strategy_cross_file", 0.92)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "lsp_direct", 0.95)); + /* A bare call (is_method=false) is a free-function call → never suppressed. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, false, "suffix_match", 0.11)); + /* Non-Go languages are never affected by this gate. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(false, true, "suffix_match", 0.11)); + /* No match (NULL/empty strategy) → nothing to suppress. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, NULL, 0.5)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "", 0.5)); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ /* Method call THROUGH an imported symbol that is itself an indexed node @@ -1072,4 +1112,6 @@ SUITE(registry) { RUN_TEST(local_binding_suppress_drops_weak_shadowed_bare_calls); RUN_TEST(local_binding_suppress_keeps_unshadowed_and_strong_strategies); RUN_TEST(weak_call_guards_share_one_drop_list); + RUN_TEST(go_suppress_drops_weak_selector_matches); + RUN_TEST(go_suppress_keeps_typed_and_import_aware_matches); } From da7619474d5ad27ff5073cf505b0b6cd5d0db36e Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Sat, 29 Aug 2026 18:04:16 +0200 Subject: [PATCH 02/19] fix(extract): receiver-qualify Go method QNs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Go method's QN was the flat package form (proj.pkg.method) — the receiver was ignored, so every same-name method in a package collided on one QN and the graph upsert kept exactly one node. Measured on a real Go repo: 20 Process/Name methods across 15 files kept 2 nodes; 9 different Task() methods fused into one chimera node carrying all nine bodies' call edges; 19 structs pointed DEFINES_METHOD at a single shared method node; a _test.go mock Close outranked the production Close in the dedupe tie-break. The upsert's own comment calls kind-disambiguated QNs 'the real cure'. Qualify the QN with the receiver type (proj.pkg.Recv.method), the same shape as Go interface members and the C++ out-of-line method path right below it in extract_func_def: - extract_defs.c: def.qualified_name = parent_class + name whenever the receiver type resolves; go_receiver_type_name becomes the shared cbm_go_receiver_type_name (exported via helpers.h) so both sides of the contract use one formula. - extract_unified.c (compute_func_qn): mirror branch for method_declaration, so method-body calls keep exact source attribution instead of degrading to File-node fallback (calls_find_source). - Consumers already agree: pxc_build_lsp_def passes the def QN and parent_class (receiver_type) verbatim into the Go LSP registries, and check_go_class_implements explicitly supports class-qualified method QNs (its path (b)). Side effect: resolve_same_module's exact module.name hash no longer matches concrete methods, which kills the conf-0.9 false edges where an interface-typed call bound to an unrelated same-package method. Fixes #1909 Signed-off-by: Ilya Brykau --- internal/cbm/extract_defs.c | 14 +++++++-- internal/cbm/extract_unified.c | 19 +++++++++++++ internal/cbm/helpers.h | 6 ++++ internal/cbm/lsp/go_lsp.c | 25 ++++++++++------ tests/test_extraction.c | 50 ++++++++++++++++++++++++++++++-- tests/test_parallel.c | 5 ++-- tests/test_pipeline.c | 52 ++++++++++++++++++++++++++++++++++ 7 files changed, 156 insertions(+), 15 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index b49147dc6..d2244c034 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3521,7 +3521,7 @@ static void set_def_complexity(CBMDefinition *def, TSNode body, const CBMLangSpe * Walks to the parameter_declaration's `type` field, unwrapping pointer_type * and generic_type, and returns the type_identifier text (e.g. "OrderService"). * Returns NULL if no type_identifier is found. */ -static char *go_receiver_type_name(CBMArena *a, TSNode recv, const char *source) { +char *cbm_go_receiver_type_name(CBMArena *a, TSNode recv, const char *source) { uint32_t nc = ts_node_child_count(recv); for (uint32_t i = 0; i < nc; i++) { TSNode child = ts_node_child(recv, i); @@ -3733,12 +3733,22 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec * (and downstream Go IMPLEMENTS/OVERRIDE) link the method to its owning * struct/type node. The parent QN must match the type's node QN, which * is computed the same way (cbm_fqn_compute on the type name). */ - char *recv_type = go_receiver_type_name(a, recv, ctx->source); + char *recv_type = cbm_go_receiver_type_name(a, recv, ctx->source); if (recv_type && recv_type[0]) { /* Must match the Go type node QN (directory-based module) so the * DEFINES_METHOD edge links the method to its owning type. */ def.parent_class = cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, recv_type, ctx->language); + /* Receiver-qualify the method QN (proj.pkg.Recv.method) — same + * shape as the C++ out-of-line path below and Go interface + * members. With the flat proj.pkg.method QN every same-name + * method in a package collided in the graph upsert: one body + * survived and the twins' call edges accreted onto it. The + * call-scope side (compute_func_qn in extract_unified.c) mirrors + * this formula, and go_lsp consumers read the def QN and + * parent_class (receiver_type) verbatim, so resolution joins + * stay exact. */ + def.qualified_name = cbm_arena_sprintf(a, "%s.%s", def.parent_class, name); } } diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index cd7f3bd28..ee13cc8fd 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -1087,6 +1087,25 @@ static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLan } } + /* Go method `func (s *Storage) Close() {...}`: the def extractor records + * this as Method "proj.pkg.Storage.Close" (receiver-qualified, mirroring + * the C++ out-of-line rule above). The call-scope QN must match — a bare + * "proj.pkg.Close" names a node that no longer exists, so every call + * inside the method body would fall back to File-node attribution + * (calls_find_source). Same ONE-formula contract as the def side: + * cbm_go_receiver_type_name + cbm_fqn_compute_source_lang. */ + if (ctx->language == CBM_LANG_GO && strcmp(ts_node_type(node), "method_declaration") == 0) { + TSNode recv = ts_node_child_by_field_name(node, TS_FIELD("receiver")); + if (!ts_node_is_null(recv)) { + char *recv_type = cbm_go_receiver_type_name(ctx->arena, recv, ctx->source); + if (recv_type && recv_type[0]) { + const char *type_qn = cbm_fqn_compute_source_lang( + ctx->arena, ctx->project, ctx->rel_path, recv_type, ctx->language); + return cbm_arena_sprintf(ctx->arena, "%s.%s", type_qn, name); + } + } + } + /* Nix: a binding's own attrpath contributes scope (`a.b.fn = …`), and the def * extractor bakes it into the def QN. Compose it identically here — otherwise * an in-body call sources to a QN one or more segments short of the def, and diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 905de04a8..08600b9b1 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -119,6 +119,12 @@ TSNode cbm_resolve_func_name(TSNode node, CBMLanguage lang); // def extractor — drift dropped the class qualifier from in-body calls (#554/#621). char *cbm_cpp_out_of_line_parent_class(CBMArena *a, TSNode node, const char *source); +/* Go: resolve a method_declaration's receiver parameter_list down to the bare + * receiver type_identifier (unwrapping pointer_type / generic_type). Shared by + * def extraction (extract_defs.c) and call-scope attribution + * (extract_unified.c) so the receiver-qualified method QN has ONE formula. */ +char *cbm_go_receiver_type_name(CBMArena *a, TSNode recv, const char *source); + // Find a child node by kind string. TSNode cbm_find_child_by_kind(TSNode parent, const char *kind); diff --git a/internal/cbm/lsp/go_lsp.c b/internal/cbm/lsp/go_lsp.c index 7fec20a9f..7d9c52621 100644 --- a/internal/cbm/lsp/go_lsp.c +++ b/internal/cbm/lsp/go_lsp.c @@ -1861,15 +1861,24 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { char* func_name = lsp_node_text(ctx, name_node); if (!func_name || !func_name[0]) return; - // Enclosing-function QN must be the BARE package.Func form (no receiver - // type segment). The textual call events (extract_unified.c) source calls - // as package_qn.func_name — methods included — and the defs pass creates - // the graph Method node under the same QN, so any other form breaks the - // caller-QN join in cbm_pipeline_find_lsp_resolution and the LSP-resolved - // call silently falls back to the registry short-name resolver. The - // receiver type still reaches the registry via the def's parent_class / - // method->receiver_type; it just does not appear in the caller QN. + // Enclosing-function QN must be EXACTLY the QN the textual call events + // (extract_unified.c compute_func_qn) and the defs pass produce, or the + // caller-QN join in cbm_pipeline_find_lsp_resolution breaks and the + // LSP-resolved call silently falls back to the registry short-name + // resolver. Since #1909 receiver-qualified Go method QNs, that shared + // form is package_qn.ReceiverType.method for methods (the ONE-formula + // contract: cbm_go_receiver_type_name) and package_qn.func for functions. ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name); + if (strcmp(ts_node_type(func_node), "method_declaration") == 0) { + TSNode recv = ts_node_child_by_field_name(func_node, "receiver", 8); + if (!ts_node_is_null(recv)) { + char *recv_type = cbm_go_receiver_type_name(ctx->arena, recv, ctx->source); + if (recv_type && recv_type[0]) { + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s.%s", ctx->package_qn, + recv_type, func_name); + } + } + } // Push function scope CBMScope* saved_scope = ctx->current_scope; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 4bdab079e..41e5a8c1c 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4715,12 +4715,12 @@ TEST(extract_go_no_filename_in_module_qn) { ASSERT_NOT_NULL(conn); ASSERT_STR_EQ(conn->qualified_name, "proj.myapp.db.Conn"); - /* Go method nodes keep a FLAT QN (module + name) with a separate - * parent_class link to the receiver type — the QN must carry the + /* Go method nodes carry a receiver-qualified QN (module + receiver type + + * name) plus the parent_class link — the QN must carry the * directory-based module and NOT the `.conn.` filename segment. */ const CBMDefinition *query = find_def_by_name(r, "Query"); ASSERT_NOT_NULL(query); - ASSERT_STR_EQ(query->qualified_name, "proj.myapp.db.Query"); + ASSERT_STR_EQ(query->qualified_name, "proj.myapp.db.Conn.Query"); ASSERT_EQ(strstr(query->qualified_name, ".conn."), NULL); /* The method's parent_class must match the type node QN (for DEFINES_METHOD). */ ASSERT_NOT_NULL(query->parent_class); @@ -5501,6 +5501,49 @@ TEST(extract_python_bare_call_flag_is_depth_independent) { PASS(); } +TEST(extract_go_method_receiver_qualified_qn) { + /* A Go method QN carries the receiver type (proj.pkg.Recv.method), same + * shape as C++ out-of-line methods and Go interface members — so two + * same-name methods on different receivers no longer collide in the + * graph upsert. Free functions keep the flat package QN. */ + CBMFileResult *r = extract("package m\n" + "type Storage struct{}\n" + "type Cache struct{}\n" + "func (s *Storage) Close() {}\n" + "func (c Cache) Close() {}\n" + "func Shutdown() {}\n", + CBM_LANG_GO, "t", "x.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + bool saw_storage = false; + bool saw_cache = false; + bool saw_free = false; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (!d->name || !d->qualified_name) { + continue; + } + if (strcmp(d->name, "Close") == 0 && strcmp(d->qualified_name, "t.Storage.Close") == 0) { + ASSERT_STR_EQ(d->label, "Method"); + ASSERT_STR_EQ(d->parent_class, "t.Storage"); + saw_storage = true; + } + if (strcmp(d->name, "Close") == 0 && strcmp(d->qualified_name, "t.Cache.Close") == 0) { + ASSERT_STR_EQ(d->parent_class, "t.Cache"); + saw_cache = true; + } + if (strcmp(d->name, "Shutdown") == 0) { + ASSERT_STR_EQ(d->qualified_name, "t.Shutdown"); + saw_free = true; + } + } + ASSERT_TRUE(saw_storage); + ASSERT_TRUE(saw_cache); + ASSERT_TRUE(saw_free); + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -7541,6 +7584,7 @@ SUITE(extraction) { RUN_TEST(extract_python_bare_call_flags_locally_bound_callee); RUN_TEST(extract_python_bare_call_flag_is_depth_independent); RUN_TEST(extract_go_selector_call_flags_is_method); + RUN_TEST(extract_go_method_receiver_qualified_qn); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 13477b67c..6ae812528 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -3444,8 +3444,9 @@ TEST(parallel_go_cross_package_field_chain_resolves) { cbm_gbuf_t *gbuf = run_go_field_chain_sequential("go_field_fold", tmpdir, files, 2); ASSERT_NOT_NULL(gbuf); - const cbm_gbuf_edge_t *edge = - find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder"); + /* Since #1909, Go method QNs are receiver-qualified on both endpoints. */ + const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment( + gbuf, "OrderHandler.PlaceOrder", ".service.OrderService.PlaceOrder"); const bool found = edge != NULL; const bool dispatch = edge && edge->properties_json && strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\""); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index dfa5fea20..9cc757b96 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5467,6 +5467,57 @@ TEST(pipeline_python_receiver_suppresses_weak_method_edge) { PASS(); } +TEST(pipeline_go_method_caller_keeps_lsp_join) { + /* #1909: receiver-qualifying method QNs moves the def/textual-call QN to + * package.Type.method — go_lsp's enclosing-function QN must move with it + * (the ONE-formula contract), or every LSP resolution sourced from inside + * a method body loses its caller join and the edge dies. The callee name + * collides across receivers so no short-name fallback can mask the loss. */ + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_mcall_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "go.mod", "module example.com/fxmcall\n\ngo 1.22\n"); + write_temp_file(tmp, "svc/repo.go", + "package svc\n" + "\n" + "type Repo struct {\n" + "\tn int\n" + "}\n" + "\n" + "func (r *Repo) Install() int {\n" + "\treturn r.helper()\n" + "}\n"); + write_temp_file(tmp, "svc/helper.go", + "package svc\n" + "\n" + "func (r *Repo) helper() int {\n" + "\treturn r.n\n" + "}\n"); + write_temp_file(tmp, "other/other.go", + "package other\n" + "\n" + "type Decoy struct{}\n" + "\n" + "func (d *Decoy) helper() int { return 2 }\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_mcall.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + ASSERT_TRUE(cross_file_call_exists(s, project, "Install", "helper")); + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_go_receiver_suppresses_weak_method_edge) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_XXXXXX"); @@ -14687,6 +14738,7 @@ SUITE(pipeline) { #endif RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_method_caller_keeps_lsp_join); RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); From d21949ef2c22a2ed099075ebb24eb26d5d71832a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 17 Oct 2026 21:14:39 +0200 Subject: [PATCH 03/19] feat(deploy): render Helm images, mark default-valued images, link crds and werf charts Four gaps of the deployment graph, closed together. Images named from a default value keep their Image; the edge says so. A Dockerfile FROM / COPY --from filled by an ARG default (an ARG chain names each ARG) carries default_of, as do compose `${X:-d}` images and `docker build -t ${IMAGE:-x}` tags. An override found in the repository adds its own edge: compose build.args and --build-arg make the Image the build names DEPEND_ON the base they choose {override_of}; compose's .env replacing an inline default marks the effective edge override_of and keeps the default as {ref: image_default}, an image the service does not run. Helm template images render (helm_template.c): a bounded Go text/template + Sprig evaluator over the vendored gotmpl grammar, with values read the way Helm reads them (go-yaml v2 scalars, float64 numbers, Helm's coalesce). Each templated container image renders from its chart family's defaults -- include / define helpers, with / range / if, subchart values under their alias, global -- and the object USES it {default_of: the values keys read}; the chart DEPLOYS it. A value that does not render (a .Release field, lookup, a missing define, a field of a missing map, a function not modelled) names no Image; a rendered value is checked against the image reference grammar first. Releases render their charts with their values: a Flux HelmRelease's spec.values, a kustomize helmCharts entry's valuesFile + valuesInline (a chart in chartHome now links locally), a Terraform helm_release's values + set (literals only; set_sensitive is never read) -- each image that differs is the release's own DEPLOYS {override_of}. A release of a chart outside the repository names images by convention only where its values' `repository` names a registry host {inferred: values}; a host-less one names none. A chart DEPLOYS the CRDs of its crds/ {phase: crds}. A werf `.helm` dir beside werf.yaml is a chart named after the werf project. Incremental runs: a werf.yaml edit and a werf chart's values / templates decline like a Chart.yaml chart's; a release's values file (a kustomize valuesFile, a Terraform file()) declines too -- Terraform's now IMPORTS {local_name: values} it, like kustomize's valuesFile. Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + internal/cbm/cbm.h | 6 +- internal/cbm/extract_k8s.c | 10 +- src/pipeline/helm_template.c | 2912 +++++++++++++++++++++++++++ src/pipeline/helm_template.h | 83 + src/pipeline/pass_ci.c | 88 +- src/pipeline/pass_docker.c | 659 +++++- src/pipeline/pass_k8s.c | 1310 +++++++++++- src/pipeline/pass_shell.c | 150 +- src/pipeline/pass_terraform.c | 399 ++++ src/pipeline/pipeline_incremental.c | 45 +- src/pipeline/pipeline_internal.h | 38 +- tests/test_grammar_probe_f.c | 271 ++- tests/test_pipeline.c | 133 ++ 14 files changed, 5976 insertions(+), 129 deletions(-) create mode 100644 src/pipeline/helm_template.c create mode 100644 src/pipeline/helm_template.h diff --git a/Makefile.cbm b/Makefile.cbm index b4b8c8827..1e3055874 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -537,6 +537,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ src/pipeline/pass_k8s.c \ + src/pipeline/helm_template.c \ src/pipeline/pass_operator.c \ src/pipeline/pass_shell.c \ src/pipeline/pass_terraform.c \ diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 86586dd15..103cc0cde 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -719,7 +719,9 @@ typedef struct { * from the URL `name`; a HelmRelease whose chartRef names it deploys the * chart there. * CBM_K8S_FACT_IMAGE: object def_index runs its container `container` - * (NULL: unnamed) from image `name` (a literal reference); via "container". + * (NULL: unnamed) from image `name` (a literal reference, or a Helm + * template's placeholder `<...>`, which names none); via "container"; + * `line` the image's line, start_line / end_line the container's. * CBM_K8S_FACT_ENV: object def_index sets environment variable `name` in * its container `container` (NULL: unnamed) at `line`; via = where the * value comes from ("value", or the valueFrom kind) -- never the value. @@ -758,6 +760,8 @@ typedef struct { const char *url; /* DEPLOY (kustomize): the chart repository */ const char *release; /* DEPLOY (kustomize): the release name */ int line; + int start_line; /* IMAGE: the container mapping's first line */ + int end_line; /* IMAGE: its last line */ } CBMK8sFact; typedef struct { diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index a3572763e..ce41a5690 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -1487,12 +1487,18 @@ static void k8s_container(k8s_walk_t *w, const char *path, TSNode container) { const char *cname = k8s_text(w, container, "name"); cname = cname && cname[0] && !strstr(cname, "{{") ? cname : NULL; const char *image = k8s_text(w, container, "image"); - if (k8s_image_literal(image)) { + /* a Helm template's image is its skeleton placeholder: pass_k8s.c renders it */ + if (k8s_image_literal(image) || (image && strchr(image, '<'))) { + TSNode iv = k8s_child(w, container, "image"); CBMK8sFact f = {.fact = CBM_K8S_FACT_IMAGE, .def_index = w->def_index, .name = image, .container = cname, - .via = "container"}; + .via = "container", + .line = + ts_node_is_null(iv) ? 0 : (int)ts_node_start_point(iv).row + SKIP_ONE, + .start_line = (int)ts_node_start_point(container).row + SKIP_ONE, + .end_line = (int)ts_node_end_point(container).row + SKIP_ONE}; cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); } k8s_iter_t env = k8s_iter(k8s_child(w, container, "env")); diff --git a/src/pipeline/helm_template.c b/src/pipeline/helm_template.c new file mode 100644 index 000000000..79293e638 --- /dev/null +++ b/src/pipeline/helm_template.c @@ -0,0 +1,2912 @@ +/* + * helm_template.c -- Helm values and the rendering of a chart template's + * image references (see helm_template.h). + * + * Values: YAML -> JSON as Helm does it (sigs.k8s.io/yaml over go-yaml v2): + * YAML 1.1 booleans (yes/no/on/off), `~` / null / empty a null, every number + * a float64 -- a float prints like Go's %v (1.10 -> 1.1, 20231010 -> + * 2.023101e+07), the way Helm renders it. --set values keep int64. + * + * Rendering: Go text/template over the vendored gotmpl grammar -- text with + * {{- / -}} trimming; if / else if / else, with, range (lists, maps by sorted + * key, else, break / continue), variables (:= / =, `$` the template's data), + * define / template / block / include / tpl, pipelines (the piped value is + * the last argument); and the Sprig functions image expressions use. A missing + * map key renders "" (Helm drops ""); a field of nil is an error. + * An error, a function not modelled, .Release.Name / .Capabilities / lookup, + * a define not found -- each is UNKNOWN, which spreads to what reads it; a + * condition or range that is UNKNOWN runs no branch. Nesting of include / + * template stops where Helm's does (1000), an error there too. + */ +#include "pipeline/helm_template.h" + +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/json_writer.h" /* cbm_json_writer_allocator */ +#include "foundation/mem_core.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include +#include + +enum { HELM_MAX_NESTING = 1000 }; /* Helm engine's recursionMaxNums */ + +/* ── Values: YAML -> JSON ─────────────────────────────────────────── */ + +typedef struct { + const char *name; + size_t len; + yyjson_mut_val *val; +} hy_anchor_t; + +typedef struct { + yyjson_mut_doc *doc; + const char *src; + hy_anchor_t *anchors; + int nanchors; + int cap; + bool unknown; +} hy_t; + +static bool hy_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *hy_text(const hy_t *y, TSNode n, size_t *len) { + uint32_t b = ts_node_start_byte(n); + *len = ts_node_end_byte(n) - b; + return y->src + b; +} + +static void hy_anchor(hy_t *y, const char *name, size_t len, yyjson_mut_val *v) { + if (y->nanchors == y->cap) { + int cap = y->cap ? y->cap * PAIR_LEN : CBM_SZ_16; + hy_anchor_t *g = cbm_realloc(CBM_MEM_CLASS_OTHER, y->anchors, (size_t)cap * sizeof(*g)); + if (!g) { + y->unknown = true; + return; + } + y->anchors = g; + y->cap = cap; + } + y->anchors[y->nanchors++] = (hy_anchor_t){name, len, v}; +} + +static yyjson_mut_val *hy_alias(hy_t *y, const char *name, size_t len) { + for (int i = y->nanchors - SKIP_ONE; i >= 0; i--) { + if (y->anchors[i].len == len && memcmp(y->anchors[i].name, name, len) == 0) { + return yyjson_mut_val_mut_copy(y->doc, y->anchors[i].val); + } + } + y->unknown = true; + return NULL; +} + +static bool hy_in(const char *s, size_t n, const char *const *set) { + for (; *set; set++) { + if (strlen(*set) == n && memcmp(*set, s, n) == 0) { + return true; + } + } + return false; +} + +/* A float of go-yaml v2's form: [-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)? */ +static bool hy_float_form(const char *s, size_t n) { + size_t i = 0; + if (i < n && (s[i] == '-' || s[i] == '+')) { + i++; + } + size_t digits = 0; + while (i < n && isdigit((unsigned char)s[i])) { + i++; + digits++; + } + if (i < n && s[i] == '.') { + i++; + size_t frac = 0; + while (i < n && isdigit((unsigned char)s[i])) { + i++; + frac++; + } + if (digits == 0 && frac == 0) { + return false; + } + } else if (digits == 0) { + return false; + } + if (i < n && (s[i] == 'e' || s[i] == 'E')) { + i++; + if (i < n && (s[i] == '-' || s[i] == '+')) { + i++; + } + size_t ed = 0; + while (i < n && isdigit((unsigned char)s[i])) { + i++; + ed++; + } + if (ed == 0) { + return false; + } + } + return i == n; +} + +/* An integer the way go-yaml v2 reads one: strconv.ParseInt(s, 0, 64) after + * dropping `_` -- decimal, 0x hex, 0b binary, a leading 0 octal. */ +static bool hy_int_form(const char *s, size_t n, int64_t *out) { + char buf[CBM_SZ_64]; + size_t k = 0; + for (size_t i = 0; i < n && k + SKIP_ONE < sizeof(buf); i++) { + if (s[i] != '_') { + buf[k++] = s[i]; + } + } + if (k == 0 || k + SKIP_ONE >= sizeof(buf)) { + return false; + } + buf[k] = '\0'; + const char *p = buf; + bool neg = false; + if (*p == '-' || *p == '+') { + neg = *p == '-'; + p++; + } + int base = 10; + if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { + base = 16; + p += PAIR_LEN; + } else if (p[0] == '0' && (p[1] == 'b' || p[1] == 'B')) { + base = 2; + p += PAIR_LEN; + } else if (p[0] == '0' && p[1]) { + base = 8; + p++; + } + if (!*p) { + return false; + } + char *end = NULL; + long long v = strtoll(p, &end, base); + if (!end || *end) { + return false; + } + *out = neg ? -(int64_t)v : (int64_t)v; + return true; +} + +static yyjson_mut_val *hy_plain(hy_t *y, const char *s, size_t n) { + static const char *const NULLS[] = {"", "~", "null", "Null", "NULL", NULL}; + static const char *const TRUES[] = {"y", "Y", "yes", "Yes", "YES", "true", + "True", "TRUE", "on", "On", "ON", NULL}; + static const char *const FALSES[] = {"n", "N", "no", "No", "NO", "false", + "False", "FALSE", "off", "Off", "OFF", NULL}; + static const char *const SPECIAL[] = {".inf", ".Inf", ".INF", "+.inf", "+.Inf", + "+.INF", "-.inf", "-.Inf", "-.INF", ".nan", + ".NaN", ".NAN", NULL}; + if (hy_in(s, n, NULLS)) { + return yyjson_mut_null(y->doc); + } + if (hy_in(s, n, TRUES)) { + return yyjson_mut_true(y->doc); + } + if (hy_in(s, n, FALSES)) { + return yyjson_mut_false(y->doc); + } + if (hy_in(s, n, SPECIAL)) { + y->unknown = true; /* JSON holds no inf / nan: Helm fails here */ + return NULL; + } + char c = n ? s[0] : '\0'; + if (isdigit((unsigned char)c) || c == '-' || c == '+' || c == '.') { + int64_t iv = 0; + if (hy_int_form(s, n, &iv)) { + return yyjson_mut_real(y->doc, (double)iv); + } + if (hy_float_form(s, n)) { + char buf[CBM_SZ_64]; + snprintf(buf, sizeof(buf), "%.*s", (int)n, s); + return yyjson_mut_real(y->doc, strtod(buf, NULL)); + } + } + return yyjson_mut_strncpy(y->doc, s, n); +} + +static int hy_hex(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + c = (char)tolower((unsigned char)c); + return c >= 'a' && c <= 'f' ? c - 'a' + 10 : -SKIP_ONE; +} + +static size_t hy_utf8(unsigned long cp, char *out) { + if (cp < 0x80) { + out[0] = (char)cp; + return SKIP_ONE; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return PAIR_LEN; + } + if (cp < 0x10000) { + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + out[0] = (char)(0xF0 | (cp >> 18)); + out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +/* Fold a quoted scalar's line breaks: a break and the blanks around it + * become one space, an empty line a newline. */ +static size_t hy_fold(char *buf, size_t n) { + size_t k = 0; + for (size_t i = 0; i < n;) { + if (buf[i] == '\n') { + while (k > 0 && (buf[k - SKIP_ONE] == ' ' || buf[k - SKIP_ONE] == '\t')) { + k--; + } + size_t breaks = 0; + while (i < n && (buf[i] == '\n' || buf[i] == ' ' || buf[i] == '\t' || buf[i] == '\r')) { + breaks += buf[i] == '\n'; + i++; + } + if (breaks > SKIP_ONE) { + for (size_t b = SKIP_ONE; b < breaks; b++) { + buf[k++] = '\n'; + } + } else { + buf[k++] = ' '; + } + continue; + } + buf[k++] = buf[i++]; + } + return k; +} + +static yyjson_mut_val *hy_double(hy_t *y, const char *s, size_t n) { + if (n < PAIR_LEN) { + y->unknown = true; + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, n * 4 + SKIP_ONE); + if (!buf) { + y->unknown = true; + return NULL; + } + size_t k = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n; i++) { + char c = s[i]; + if (c != '\\') { + buf[k++] = c; + continue; + } + if (++i + SKIP_ONE >= n + SKIP_ONE) { + break; + } + c = s[i]; + static const char ESC_IN[] = "0abtnvfre \"/\\N_LP\t"; + static const char ESC_OUT[] = "\0\a\b\t\n\v\f\r\x1b \"/\\"; + const char *e = strchr(ESC_IN, c); + if (c == '\n') { + while (i + SKIP_ONE < n && (s[i + SKIP_ONE] == ' ' || s[i + SKIP_ONE] == '\t')) { + i++; + } + } else if (e && (size_t)(e - ESC_IN) < sizeof(ESC_OUT) - SKIP_ONE) { + buf[k++] = ESC_OUT[e - ESC_IN]; + } else if (c == 'x' || c == 'u' || c == 'U') { + int digits = c == 'x' ? PAIR_LEN : c == 'u' ? 4 : 8; + uint32_t cp = 0; + for (int d = 0; d < digits; d++) { + int h = i + SKIP_ONE < n ? hy_hex(s[i + SKIP_ONE]) : -SKIP_ONE; + if (h < 0) { + y->unknown = true; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return NULL; + } + cp = cp * 16 + (uint32_t)h; + i++; + } + k += hy_utf8(cp, buf + k); + } else { + y->unknown = true; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return NULL; + } + } + k = hy_fold(buf, k); + yyjson_mut_val *v = yyjson_mut_strncpy(y->doc, buf, k); + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return v; +} + +static yyjson_mut_val *hy_single(hy_t *y, const char *s, size_t n) { + if (n < PAIR_LEN) { + y->unknown = true; + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!buf) { + y->unknown = true; + return NULL; + } + size_t k = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n; i++) { + buf[k++] = s[i]; + if (s[i] == '\'' && i + PAIR_LEN < n && s[i + SKIP_ONE] == '\'') { + i++; + } + } + k = hy_fold(buf, k); + yyjson_mut_val *v = yyjson_mut_strncpy(y->doc, buf, k); + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return v; +} + +/* A block scalar (| or >, chomping - / +, an explicit indentation digit). */ +static yyjson_mut_val *hy_block(hy_t *y, const char *s, size_t n) { + size_t i = 0; + bool folded = n > 0 && s[0] == '>'; + char chomp = 'c'; + int explicit_indent = 0; + for (i = SKIP_ONE; i < n && s[i] != '\n' && s[i] != '#'; i++) { + if (s[i] == '-' || s[i] == '+') { + chomp = s[i]; + } else if (isdigit((unsigned char)s[i])) { + explicit_indent = s[i] - '0'; + } + } + while (i < n && s[i] != '\n') { + i++; + } + i += i < n ? SKIP_ONE : 0; + /* the content's indentation: the explicit one, else its first non-empty line's */ + size_t indent = 0; + if (explicit_indent > 0) { + /* relative to the parent's: the line indentation of the scalar's own start */ + const char *ls = s; + while (ls > y->src && ls[-1] != '\n') { + ls--; + } + size_t parent = 0; + while (ls + parent < s && ls[parent] == ' ') { + parent++; + } + indent = parent + (size_t)explicit_indent; + } else { + for (size_t j = i; j < n;) { + size_t sp = 0; + while (j + sp < n && s[j + sp] == ' ') { + sp++; + } + if (j + sp < n && s[j + sp] != '\n') { + indent = sp; + break; + } + j += sp + SKIP_ONE; + } + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, n + PAIR_LEN); + if (!buf) { + y->unknown = true; + return NULL; + } + size_t k = 0; + bool prev_more = false; /* the previous line was more indented (folded keeps its break) */ + bool first = true; + while (i < n) { + size_t sp = 0; + while (i + sp < n && s[i + sp] == ' ' && sp < indent) { + sp++; + } + size_t e = i + sp; + while (e < n && s[e] != '\n') { + e++; + } + bool empty = e == i + sp; + bool more = !empty && (s[i + sp] == ' ' || s[i + sp] == '\t'); + if (folded && !first && k > 0 && buf[k - SKIP_ONE] == '\n' && !empty && !more && + !prev_more) { + /* a break between two plain lines folds into a space ... */ + size_t back = k - SKIP_ONE; + bool only = back == 0 || buf[back - SKIP_ONE] != '\n'; + if (only) { + buf[back] = ' '; + } else { + /* ... one of several breaks is dropped */ + k = back; + } + } + memcpy(buf + k, s + i + sp, e - (i + sp)); + k += e - (i + sp); + buf[k++] = '\n'; + prev_more = more; + first = empty && first; + i = e + SKIP_ONE; + } + if (chomp != '+') { + while (k > 0 && buf[k - SKIP_ONE] == '\n') { + k--; + } + if (chomp == 'c' && k > 0) { + buf[k++] = '\n'; + } + } + yyjson_mut_val *v = yyjson_mut_strncpy(y->doc, buf, k); + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return v; +} + +static yyjson_mut_val *hy_node(hy_t *y, TSNode n); + +/* A mapping key as JSON: its scalar's text (sigs.k8s.io/yaml stringifies). */ +static yyjson_mut_val *hy_key(hy_t *y, TSNode k) { + yyjson_mut_val *v = ts_node_is_null(k) ? yyjson_mut_null(y->doc) : hy_node(y, k); + if (!v) { + return NULL; + } + if (yyjson_mut_is_str(v)) { + return v; + } + if (yyjson_mut_is_null(v)) { + return yyjson_mut_strcpy(y->doc, "null"); + } + if (yyjson_mut_is_bool(v)) { + return yyjson_mut_strcpy(y->doc, yyjson_mut_get_bool(v) ? "true" : "false"); + } + if (yyjson_mut_is_real(v)) { + /* go-yaml v2 keeps an integer key an int: its decimal digits */ + double d = yyjson_mut_get_real(v); + char buf[CBM_SZ_64]; + if (d == floor(d) && fabs(d) < 1e18) { + snprintf(buf, sizeof(buf), "%lld", (long long)d); + } else { + snprintf(buf, sizeof(buf), "%.17g", d); + } + return yyjson_mut_strcpy(y->doc, buf); + } + y->unknown = true; /* a mapping / sequence as a key */ + return NULL; +} + +/* YAML merge key: `<<: *a` / `<<: [*a, *b]` -- entries not already there. */ +static void hy_merge(yyjson_mut_val *obj, yyjson_mut_val *src) { + if (yyjson_mut_is_arr(src)) { + size_t idx, max; + yyjson_mut_val *it; + yyjson_mut_arr_foreach(src, idx, max, it) { + hy_merge(obj, it); + } + return; + } + if (!yyjson_mut_is_obj(src)) { + return; + } + size_t idx, max; + yyjson_mut_val *k, *v; + yyjson_mut_obj_foreach(src, idx, max, k, v) { + if (!yyjson_mut_obj_getn(obj, yyjson_mut_get_str(k), yyjson_mut_get_len(k))) { + yyjson_mut_obj_add(obj, k, v); + } + } +} + +static yyjson_mut_val *hy_mapping(hy_t *y, TSNode n, const char *pair_type) { + yyjson_mut_val *obj = yyjson_mut_obj(y->doc); + yyjson_mut_val *merges = NULL; + for (uint32_t i = 0; obj && i < ts_node_named_child_count(n); i++) { + TSNode p = ts_node_named_child(n, i); + if (hy_type(p, "comment")) { + continue; + } + TSNode kn = p; + TSNode vn = {0}; + if (hy_type(p, pair_type)) { + kn = ts_node_child_by_field_name(p, "key", (uint32_t)strlen("key")); + vn = ts_node_child_by_field_name(p, "value", (uint32_t)strlen("value")); + } else if (!hy_type(p, "flow_node")) { + y->unknown = true; + return NULL; + } + size_t kl = 0; + const char *kt = ts_node_is_null(kn) ? "" : hy_text(y, kn, &kl); + yyjson_mut_val *val = ts_node_is_null(vn) ? yyjson_mut_null(y->doc) : hy_node(y, vn); + if (!val || y->unknown) { + y->unknown = true; + return NULL; + } + if (kl == PAIR_LEN && memcmp(kt, "<<", PAIR_LEN) == 0) { + merges = merges ? merges : yyjson_mut_arr(y->doc); + yyjson_mut_arr_append(merges, val); + continue; + } + yyjson_mut_val *key = hy_key(y, kn); + if (!key) { + y->unknown = true; + return NULL; + } + (void)yyjson_mut_obj_put(obj, key, val); /* a duplicate key: the last one */ + } + if (obj && merges) { + hy_merge(obj, merges); + } + return obj; +} + +static yyjson_mut_val *hy_sequence(hy_t *y, TSNode n) { + yyjson_mut_val *arr = yyjson_mut_arr(y->doc); + for (uint32_t i = 0; arr && i < ts_node_named_child_count(n); i++) { + TSNode it = ts_node_named_child(n, i); + if (hy_type(it, "comment")) { + continue; + } + TSNode content = it; + if (hy_type(it, "block_sequence_item")) { + content = ts_node_named_child_count(it) > 0 ? ts_node_named_child(it, 0) : (TSNode){0}; + while (!ts_node_is_null(content) && hy_type(content, "comment")) { + content = ts_node_next_named_sibling(content); + } + } + yyjson_mut_val *v = ts_node_is_null(content) ? yyjson_mut_null(y->doc) + : hy_type(content, "flow_pair") ? NULL + : hy_node(y, content); + if (hy_type(content, "flow_pair")) { + /* [a: b] -- a single-pair map */ + v = yyjson_mut_obj(y->doc); + TSNode kn = ts_node_child_by_field_name(content, "key", (uint32_t)strlen("key")); + TSNode vn = ts_node_child_by_field_name(content, "value", (uint32_t)strlen("value")); + yyjson_mut_val *key = hy_key(y, kn); + yyjson_mut_val *val = ts_node_is_null(vn) ? yyjson_mut_null(y->doc) : hy_node(y, vn); + if (!key || !val) { + y->unknown = true; + return NULL; + } + yyjson_mut_obj_add(v, key, val); + } + if (!v || y->unknown) { + y->unknown = true; + return NULL; + } + yyjson_mut_arr_append(arr, v); + } + return arr; +} + +static yyjson_mut_val *hy_node(hy_t *y, TSNode n) { + if (ts_node_is_null(n) || ts_node_has_error(n) || y->unknown) { + y->unknown = true; + return NULL; + } + const char *t = ts_node_type(n); + if (strcmp(t, "block_node") == 0 || strcmp(t, "flow_node") == 0) { + TSNode content = {0}; + const char *anchor = NULL; + size_t al = 0; + bool str_tag = false; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (hy_type(c, "anchor")) { + TSNode nm = ts_node_named_child(c, 0); + anchor = ts_node_is_null(nm) ? NULL : hy_text(y, nm, &al); + } else if (hy_type(c, "tag")) { + size_t tl = 0; + const char *tt = hy_text(y, c, &tl); + str_tag = tl == strlen("!!str") && memcmp(tt, "!!str", tl) == 0; + } else if (!hy_type(c, "comment")) { + content = c; + } + } + yyjson_mut_val *v = NULL; + if (ts_node_is_null(content)) { + v = yyjson_mut_null(y->doc); + } else if (str_tag && hy_type(content, "plain_scalar")) { + size_t l = 0; + const char *s = hy_text(y, content, &l); + v = yyjson_mut_strncpy(y->doc, s, l); + } else { + v = hy_node(y, content); + } + if (v && anchor) { + hy_anchor(y, anchor, al, v); + } + return v; + } + size_t l = 0; + const char *s = hy_text(y, n, &l); + if (strcmp(t, "block_mapping") == 0) { + return hy_mapping(y, n, "block_mapping_pair"); + } + if (strcmp(t, "flow_mapping") == 0) { + return hy_mapping(y, n, "flow_pair"); + } + if (strcmp(t, "block_sequence") == 0 || strcmp(t, "flow_sequence") == 0) { + return hy_sequence(y, n); + } + if (strcmp(t, "plain_scalar") == 0) { + return hy_plain(y, s, l); + } + if (strcmp(t, "double_quote_scalar") == 0) { + return hy_double(y, s, l); + } + if (strcmp(t, "single_quote_scalar") == 0) { + return hy_single(y, s, l); + } + if (strcmp(t, "block_scalar") == 0) { + return hy_block(y, s, l); + } + if (strcmp(t, "alias") == 0) { + TSNode nm = ts_node_named_child(n, 0); + size_t nl = 0; + const char *name = ts_node_is_null(nm) ? NULL : hy_text(y, nm, &nl); + return name ? hy_alias(y, name, nl) : NULL; + } + y->unknown = true; + return NULL; +} + +static void hy_done(hy_t *y) { + cbm_free(CBM_MEM_CLASS_OTHER, y->anchors); + y->anchors = NULL; +} + +yyjson_mut_val *cbm_helm_values_node(yyjson_mut_doc *doc, const char *src, TSNode node) { + hy_t y = {.doc = doc, .src = src}; + yyjson_mut_val *v = hy_node(&y, node); + hy_done(&y); + return y.unknown ? NULL : v; +} + +yyjson_mut_val *cbm_helm_values_parse(yyjson_mut_doc *doc, const char *src, size_t len) { + TSParser *parser = ts_parser_new(); + if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) { + if (parser) { + ts_parser_delete(parser); + } + return NULL; + } + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + yyjson_mut_val *out = NULL; + if (tree) { + TSNode root = ts_tree_root_node(tree); + TSNode docn = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { + if (hy_type(ts_node_named_child(root, i), "document")) { + docn = ts_node_named_child(root, i); + break; + } + } + TSNode content = {0}; + for (uint32_t i = 0; !ts_node_is_null(docn) && i < ts_node_named_child_count(docn); i++) { + TSNode c = ts_node_named_child(docn, i); + if (!hy_type(c, "comment")) { + content = c; + break; + } + } + if (ts_node_has_error(root) && (ts_node_is_null(docn) || ts_node_has_error(docn))) { + out = NULL; + } else if (ts_node_is_null(content)) { + out = yyjson_mut_obj(doc); + } else { + out = cbm_helm_values_node(doc, src, content); + if (out && yyjson_mut_is_null(out)) { + out = yyjson_mut_obj(doc); + } + } + ts_tree_delete(tree); + } + ts_parser_delete(parser); + return out; +} + +static yyjson_mut_val *hc_merge(yyjson_mut_doc *doc, yyjson_mut_val *dst, yyjson_mut_val *over) { + if (!yyjson_mut_is_obj(dst) || !yyjson_mut_is_obj(over)) { + return over ? yyjson_mut_val_mut_copy(doc, over) : dst; + } + size_t idx, max; + yyjson_mut_val *k, *v; + yyjson_mut_obj_foreach(over, idx, max, k, v) { + const char *ks = yyjson_mut_get_str(k); + size_t kl = yyjson_mut_get_len(k); + if (yyjson_mut_is_null(v)) { + (void)yyjson_mut_obj_remove_keyn(dst, ks, kl); + continue; + } + yyjson_mut_val *cur = yyjson_mut_obj_getn(dst, ks, kl); + yyjson_mut_val *merged = cur && yyjson_mut_is_obj(cur) && yyjson_mut_is_obj(v) + ? hc_merge(doc, cur, v) + : yyjson_mut_val_mut_copy(doc, v); + if (merged != cur) { + (void)yyjson_mut_obj_remove_keyn(dst, ks, kl); + yyjson_mut_obj_add(dst, yyjson_mut_strncpy(doc, ks, kl), merged); + } + } + return dst; +} + +yyjson_mut_val *cbm_helm_values_coalesce(yyjson_mut_doc *doc, yyjson_mut_val *base, + yyjson_mut_val *over) { + yyjson_mut_val *out = base ? yyjson_mut_val_mut_copy(doc, base) : yyjson_mut_obj(doc); + return over ? hc_merge(doc, out, over) : out; +} + +/* Helm strvals' typedVal. */ +static yyjson_mut_val *hs_typed(yyjson_mut_doc *doc, const char *v, bool as_string) { + if (as_string) { + return yyjson_mut_strcpy(doc, v); + } + if (strcasecmp(v, "true") == 0) { + return yyjson_mut_true(doc); + } + if (strcasecmp(v, "false") == 0) { + return yyjson_mut_false(doc); + } + if (strcasecmp(v, "null") == 0) { + return yyjson_mut_null(doc); + } + if (strcmp(v, "0") == 0) { + return yyjson_mut_sint(doc, 0); + } + if (v[0] && v[0] != '0') { + char *end = NULL; + long long iv = strtoll(v, &end, 10); + if (end && !*end && (isdigit((unsigned char)v[0]) || v[0] == '-' || v[0] == '+')) { + return yyjson_mut_sint(doc, iv); + } + } + return yyjson_mut_strcpy(doc, v); +} + +bool cbm_helm_values_set(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *name, + const char *value, bool as_string) { + if (!yyjson_mut_is_obj(root) || !name || !name[0] || !value) { + return false; + } + yyjson_mut_val *cur = root; + const char *p = name; + char key[CBM_SZ_512]; + while (*p) { + size_t k = 0; + while (*p && *p != '.' && *p != '[' && k + SKIP_ONE < sizeof(key)) { + if (*p == '\\' && p[1]) { + p++; + } + key[k++] = *p++; + } + key[k] = '\0'; + long index = -SKIP_ONE; + if (*p == '[') { + char *end = NULL; + index = strtol(p + SKIP_ONE, &end, 10); + if (!end || *end != ']' || index < 0) { + return false; + } + p = end + SKIP_ONE; + } + bool last = !*p || (*p == '.' && !p[1]); + if (*p == '.') { + p++; + } + if (k == 0 || !yyjson_mut_is_obj(cur)) { + return false; + } + yyjson_mut_val *child = yyjson_mut_obj_get(cur, key); + if (index >= 0) { + if (!yyjson_mut_is_arr(child)) { + child = yyjson_mut_arr(doc); + (void)yyjson_mut_obj_remove_key(cur, key); + yyjson_mut_obj_add(cur, yyjson_mut_strcpy(doc, key), child); + } + while (yyjson_mut_arr_size(child) <= (size_t)index) { + yyjson_mut_arr_append(child, yyjson_mut_null(doc)); + } + if (last) { + (void)yyjson_mut_arr_replace(child, (size_t)index, hs_typed(doc, value, as_string)); + return true; + } + yyjson_mut_val *el = yyjson_mut_arr_get(child, (size_t)index); + if (!yyjson_mut_is_obj(el)) { + el = yyjson_mut_obj(doc); + (void)yyjson_mut_arr_replace(child, (size_t)index, el); + } + cur = el; + continue; + } + if (last) { + (void)yyjson_mut_obj_remove_key(cur, key); + yyjson_mut_obj_add(cur, yyjson_mut_strcpy(doc, key), hs_typed(doc, value, as_string)); + return true; + } + if (!yyjson_mut_is_obj(child)) { + child = yyjson_mut_obj(doc); + (void)yyjson_mut_obj_remove_key(cur, key); + yyjson_mut_obj_add(cur, yyjson_mut_strcpy(doc, key), child); + } + cur = child; + } + return false; +} + +/* ── Defines ──────────────────────────────────────────────────────── */ + +typedef struct { + char *rel; + char *src; + size_t len; + TSTree *tree; +} hd_file_t; + +typedef struct { + const hd_file_t *file; + TSNode node; /* the define_action / block_action */ + int depth; +} hd_def_t; + +struct cbm_helm_defines { + hd_file_t **files; /* each allocated once: defines point into them */ + int nfiles; + int cap; + CBMHashTable *names; /* name -> hd_def_t* */ +}; + +cbm_helm_defines_t *cbm_helm_defines_new(void) { + cbm_helm_defines_t *d = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*d)); + if (d) { + d->names = cbm_ht_create(0); + if (!d->names) { + cbm_free(CBM_MEM_CLASS_OTHER, d); + return NULL; + } + } + return d; +} + +static int hd_depth(const char *rel) { + int n = 0; + for (const char *p = rel; *p; p++) { + n += *p == '/'; + } + return n; +} + +/* A Go string literal's value ("..." with escapes, `...` raw); NULL: invalid. */ +static char *hg_string(const char *s, size_t n, size_t *out_len); + +static void hd_scan(cbm_helm_defines_t *d, hd_file_t *f, TSNode n, int depth) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + const char *t = ts_node_type(c); + bool def = strcmp(t, "define_action") == 0; + bool block = strcmp(t, "block_action") == 0; + if (def || block) { + TSNode nm = ts_node_child_by_field_name(c, "name", (uint32_t)strlen("name")); + size_t nl = 0; + char *name = ts_node_is_null(nm) + ? NULL + : hg_string(f->src + ts_node_start_byte(nm), + ts_node_end_byte(nm) - ts_node_start_byte(nm), &nl); + if (name) { + hd_def_t *old = cbm_ht_get(d->names, name); + bool wins = !old || depth < old->depth || + (depth == old->depth && strcmp(f->rel, old->file->rel) < 0); + hd_def_t *nd = wins ? cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*nd)) : NULL; + if (nd) { + *nd = (hd_def_t){f, c, depth}; + if (old) { + const char *key = cbm_ht_get_key(d->names, name); + cbm_ht_set(d->names, key, nd); + cbm_free(CBM_MEM_CLASS_OTHER, old); + cbm_free(CBM_MEM_CLASS_OTHER, name); + } else { + cbm_ht_set(d->names, name, nd); + } + name = NULL; + } + cbm_free(CBM_MEM_CLASS_OTHER, name); + } + } + if (block || strcmp(t, "if_action") == 0 || strcmp(t, "with_action") == 0 || + strcmp(t, "range_action") == 0) { + hd_scan(d, f, c, depth); /* a define is top-level; a block nests */ + } + } +} + +void cbm_helm_defines_add(cbm_helm_defines_t *d, const char *rel, const char *src, size_t len) { + if (!d || !src) { + return; + } + if (d->nfiles == d->cap) { + int cap = d->cap ? d->cap * PAIR_LEN : CBM_SZ_16; + hd_file_t **g = cbm_realloc(CBM_MEM_CLASS_OTHER, d->files, (size_t)cap * sizeof(*g)); + if (!g) { + return; + } + d->files = g; + d->cap = cap; + } + hd_file_t *f = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*f)); + if (!f) { + return; + } + f->rel = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, rel); + f->src = cbm_alloc(CBM_MEM_CLASS_OTHER, len + SKIP_ONE); + if (!f->rel || !f->src) { + cbm_free(CBM_MEM_CLASS_OTHER, f->rel); + cbm_free(CBM_MEM_CLASS_OTHER, f->src); + cbm_free(CBM_MEM_CLASS_OTHER, f); + return; + } + memcpy(f->src, src, len); + f->src[len] = '\0'; + f->len = len; + TSParser *parser = ts_parser_new(); + if (parser && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_GOTEMPLATE))) { + f->tree = ts_parser_parse_string(parser, NULL, f->src, (uint32_t)len); + } + if (parser) { + ts_parser_delete(parser); + } + d->files[d->nfiles++] = f; + if (f->tree) { + hd_scan(d, f, ts_tree_root_node(f->tree), hd_depth(rel)); + } +} + +static void hd_free_def(const char *key, void *value, void *ud) { + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); + cbm_free(CBM_MEM_CLASS_OTHER, value); +} + +void cbm_helm_defines_free(cbm_helm_defines_t *d) { + if (!d) { + return; + } + if (d->names) { + cbm_ht_foreach(d->names, hd_free_def, NULL); + cbm_ht_free(d->names); + } + for (int i = 0; i < d->nfiles; i++) { + if (d->files[i]->tree) { + ts_tree_delete(d->files[i]->tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, d->files[i]->rel); + cbm_free(CBM_MEM_CLASS_OTHER, d->files[i]->src); + cbm_free(CBM_MEM_CLASS_OTHER, d->files[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, d->files); + cbm_free(CBM_MEM_CLASS_OTHER, d); +} + +static const hd_def_t *hd_find(const cbm_helm_defines_t *d, const char *name) { + return d && d->names ? cbm_ht_get(d->names, name) : NULL; +} + +/* ── Go literals and formatting ───────────────────────────────────── */ + +/* The value of Go string literal s ("..." or `...`) into out (capacity n); + * its length, or (size_t)-1 when it is no valid literal. */ +static size_t hg_unquote(const char *s, size_t n, char *out) { + if (n >= PAIR_LEN && s[0] == '`' && s[n - SKIP_ONE] == '`') { + memcpy(out, s + SKIP_ONE, n - PAIR_LEN); + return n - PAIR_LEN; + } + if (n < PAIR_LEN || s[0] != '"' || s[n - SKIP_ONE] != '"') { + return (size_t)-1; + } + size_t k = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n; i++) { + char c = s[i]; + if (c != '\\') { + out[k++] = c; + continue; + } + if (++i + SKIP_ONE >= n) { + return (size_t)-1; + } + c = s[i]; + static const char IN[] = "abfnrtv\\\"'"; + static const char OUT[] = "\a\b\f\n\r\t\v\\\"'"; + const char *e = strchr(IN, c); + if (e && c) { + out[k++] = OUT[e - IN]; + } else if (c == 'x' || c == 'u' || c == 'U') { + int digits = c == 'x' ? PAIR_LEN : c == 'u' ? 4 : 8; + uint32_t cp = 0; + for (int d = 0; d < digits; d++) { + int h = i + SKIP_ONE + SKIP_ONE < n ? hy_hex(s[i + SKIP_ONE]) : -SKIP_ONE; + if (h < 0) { + return (size_t)-1; + } + cp = cp * 16 + (uint32_t)h; + i++; + } + if (c == 'x') { + out[k++] = (char)cp; + } else { + k += hy_utf8(cp, out + k); + } + } else if (c >= '0' && c <= '7') { + uint32_t v = 0; + for (int d = 0; d < 3 && i + SKIP_ONE < n && s[i] >= '0' && s[i] <= '7'; d++, i++) { + v = v * 8 + (uint32_t)(s[i] - '0'); + } + i--; + out[k++] = (char)v; + } else { + return (size_t)-1; + } + } + return k; +} + +static char *hg_string(const char *s, size_t n, size_t *out_len) { + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, n * 4 + SKIP_ONE); + if (!out) { + return NULL; + } + size_t k = hg_unquote(s, n, out); + if (k == (size_t)-1) { + cbm_free(CBM_MEM_CLASS_OTHER, out); + return NULL; + } + out[k] = '\0'; + *out_len = k; + return out; +} + +/* float64 as Go's %v writes it (strconv 'g', shortest). */ +static void hg_float(double f, char *out, size_t sz) { + if (f == 0) { + snprintf(out, sz, signbit(f) ? "-0" : "0"); + return; + } + char e[CBM_SZ_64]; + int prec = 0; + for (prec = 0; prec < 17; prec++) { + snprintf(e, sizeof(e), "%.*e", prec, f); + if (strtod(e, NULL) == f) { + break; + } + } + /* e: [-]d[.ddd]e(+|-)XX */ + bool neg = e[0] == '-'; + const char *p = e + (neg ? SKIP_ONE : 0); + char digs[CBM_SZ_32]; + int nd = 0; + for (; *p && *p != 'e'; p++) { + if (isdigit((unsigned char)*p) && nd + SKIP_ONE < (int)sizeof(digs)) { + digs[nd++] = *p; + } + } + while (nd > SKIP_ONE && digs[nd - SKIP_ONE] == '0') { + nd--; + } + digs[nd] = '\0'; + int exp = *p == 'e' ? atoi(p + SKIP_ONE) : 0; + int dp = exp + SKIP_ONE; + size_t k = 0; + if (neg) { + out[k++] = '-'; + } + if (exp < -4 || exp >= 6) { + out[k++] = digs[0]; + if (nd > SKIP_ONE) { + out[k++] = '.'; + for (int i = SKIP_ONE; i < nd && k + 8 < sz; i++) { + out[k++] = digs[i]; + } + } + snprintf(out + k, sz - k, "e%c%02d", exp < 0 ? '-' : '+', exp < 0 ? -exp : exp); + return; + } + if (dp <= 0) { + out[k++] = '0'; + out[k++] = '.'; + for (int i = 0; i < -dp && k + 4 < sz; i++) { + out[k++] = '0'; + } + for (int i = 0; i < nd && k + 4 < sz; i++) { + out[k++] = digs[i]; + } + } else { + for (int i = 0; i < dp && k + 4 < sz; i++) { + out[k++] = i < nd ? digs[i] : '0'; + } + if (nd > dp) { + out[k++] = '.'; + for (int i = dp; i < nd && k + 4 < sz; i++) { + out[k++] = digs[i]; + } + } + } + out[k] = '\0'; +} + +bool cbm_helm_scalar_text(yyjson_mut_val *v, char *out, size_t sz) { + if (yyjson_mut_is_str(v)) { + snprintf(out, sz, "%s", yyjson_mut_get_str(v)); + } else if (yyjson_mut_is_real(v)) { + hg_float(yyjson_mut_get_real(v), out, sz); + } else if (yyjson_mut_is_int(v)) { + snprintf(out, sz, "%lld", + yyjson_mut_is_sint(v) ? (long long)yyjson_mut_get_sint(v) + : (long long)yyjson_mut_get_uint(v)); + } else if (yyjson_mut_is_bool(v)) { + snprintf(out, sz, "%s", yyjson_mut_get_bool(v) ? "true" : "false"); + } else { + return false; + } + return true; +} + +/* ── Runtime values ───────────────────────────────────────────────── */ + +typedef enum { + HV_UNK = 0, + HV_NIL, + HV_BOOL, + HV_INT, + HV_FLOAT, + HV_STR, + HV_JSON, + HV_MAP, + HV_LIST +} hv_kind_t; + +typedef struct hv hv_t; + +typedef struct { + const char **keys; + hv_t *vals; + int n; + bool strict; /* a struct (.Chart, .Release): a field not held is UNKNOWN */ +} hv_map_t; + +typedef struct { + hv_t *items; + int n; +} hv_list_t; + +struct hv { + hv_kind_t k; + bool b; + int64_t i; + double f; + const char *s; + size_t sn; + yyjson_mut_val *j; /* HV_JSON: an object or array of the values */ + hv_map_t *m; + hv_list_t *l; + const char *path; /* the values path it holds; NULL: none */ +}; + +static const hv_t HV_UNKNOWN = {.k = HV_UNK}; + +static hv_t hv_nil(void) { + return (hv_t){.k = HV_NIL}; +} + +static hv_t hv_bool(bool b) { + return (hv_t){.k = HV_BOOL, .b = b}; +} + +static hv_t hv_int(int64_t i) { + return (hv_t){.k = HV_INT, .i = i}; +} + +static hv_t hv_float(double f) { + return (hv_t){.k = HV_FLOAT, .f = f}; +} + +static hv_t hv_str(const char *s, size_t n) { + return (hv_t){.k = HV_STR, .s = s, .sn = n}; +} + +static hv_t hv_json(yyjson_mut_val *v, const char *path) { + hv_t out = {.path = path}; + if (!v || yyjson_mut_is_null(v)) { + out.k = HV_NIL; + } else if (yyjson_mut_is_bool(v)) { + out.k = HV_BOOL; + out.b = yyjson_mut_get_bool(v); + } else if (yyjson_mut_is_int(v)) { + out.k = HV_INT; + out.i = yyjson_mut_is_sint(v) ? yyjson_mut_get_sint(v) : (int64_t)yyjson_mut_get_uint(v); + } else if (yyjson_mut_is_real(v)) { + out.k = HV_FLOAT; + out.f = yyjson_mut_get_real(v); + } else if (yyjson_mut_is_str(v)) { + out.k = HV_STR; + out.s = yyjson_mut_get_str(v); + out.sn = yyjson_mut_get_len(v); + } else if (yyjson_mut_is_obj(v) || yyjson_mut_is_arr(v)) { + out.k = HV_JSON; + out.j = v; + } + return out; +} + +static bool hv_is_map(const hv_t *v) { + return v->k == HV_MAP || (v->k == HV_JSON && yyjson_mut_is_obj(v->j)); +} + +static bool hv_is_list(const hv_t *v) { + return v->k == HV_LIST || (v->k == HV_JSON && yyjson_mut_is_arr(v->j)); +} + +static int hv_len(const hv_t *v) { + switch (v->k) { + case HV_STR: + return (int)v->sn; + case HV_MAP: + return v->m->n; + case HV_LIST: + return v->l->n; + case HV_JSON: + return (int)(yyjson_mut_is_obj(v->j) ? yyjson_mut_obj_size(v->j) + : yyjson_mut_arr_size(v->j)); + default: + return -SKIP_ONE; + } +} + +/* Go template truth (if / with / and / or / not). */ +static bool hv_truth(const hv_t *v) { + switch (v->k) { + case HV_BOOL: + return v->b; + case HV_INT: + return v->i != 0; + case HV_FLOAT: + return v->f != 0; + case HV_STR: + case HV_MAP: + case HV_LIST: + case HV_JSON: + return hv_len(v) > 0; + default: + return false; + } +} + +/* ── Renderer state ───────────────────────────────────────────────── */ + +typedef struct { + const char *name; + size_t nl; + hv_t v; +} hvar_t; + +typedef struct { + char *buf; + size_t n; + size_t cap; + bool unk; +} hsink_t; + +typedef struct { + const char *src; /* the text these nodes are of */ + hv_t dot; + int var_base; /* the template's first variable (`$`) */ + hsink_t *out; /* NULL: the template's own output, which the probes read */ +} hframe_t; + +typedef struct { + CBMArena *a; + yyjson_mut_doc *scratch; + const cbm_helm_defines_t *defs; + hvar_t *vars; + int nvars; + int vcap; + int nesting; + bool brk; + bool cont; + /* probes of the top-level template */ + const char *top_src; + size_t top_len; + cbm_helm_probe_t *probes; + int nprobes; + const uint32_t *lines; /* line start offsets */ + int nlines; + int capture; /* the probe capturing; -1: none */ + hsink_t cap; + const char **ckeys; + int nck; + int ckcap; + TSParser *parser; + TSTree **trees; + int ntrees; + int tcap; +} hst_t; + +static void hs_append(hst_t *st, hsink_t *s, const char *p, size_t n) { + if (s->unk || n == 0) { + return; + } + if (s->n + n + SKIP_ONE > s->cap) { + size_t cap = (s->cap ? s->cap : CBM_SZ_64) * PAIR_LEN; + while (cap < s->n + n + SKIP_ONE) { + cap *= PAIR_LEN; + } + char *g = cbm_arena_alloc(st->a, cap); + if (!g) { + s->unk = true; + return; + } + if (s->n) { + memcpy(g, s->buf, s->n); + } + s->buf = g; + s->cap = cap; + } + memcpy(s->buf + s->n, p, n); + s->n += n; + s->buf[s->n] = '\0'; +} + +static const char *hs_str(hst_t *st, const hsink_t *s) { + return s->n ? s->buf : cbm_arena_strdup(st->a, ""); +} + +static void hst_record(hst_t *st, const char *path) { + if (st->capture < 0 || !path) { + return; + } + for (int i = 0; i < st->nck; i++) { + if (strcmp(st->ckeys[i], path) == 0) { + return; + } + } + if (st->nck == st->ckcap) { + int cap = st->ckcap ? st->ckcap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + return; + } + if (st->nck) { + memcpy(g, st->ckeys, (size_t)st->nck * sizeof(*g)); + } + st->ckeys = g; + st->ckcap = cap; + } + st->ckeys[st->nck++] = path; +} + +/* A values path joined with key `k` (its own `.` escaped). */ +static const char *hst_path(hst_t *st, const char *path, const char *k, size_t kl) { + if (!path) { + return NULL; + } + size_t dots = 0; + for (size_t i = 0; i < kl; i++) { + dots += k[i] == '.'; + } + size_t pl = strlen(path); + char *out = cbm_arena_alloc(st->a, pl + kl + dots + PAIR_LEN); + if (!out) { + return NULL; + } + size_t o = 0; + if (pl) { + memcpy(out, path, pl); + o = pl; + out[o++] = '.'; + } + for (size_t i = 0; i < kl; i++) { + if (k[i] == '.') { + out[o++] = '\\'; + } + out[o++] = k[i]; + } + out[o] = '\0'; + return out; +} + +static bool hst_var_push(hst_t *st, const char *name, size_t nl, hv_t v) { + if (st->nvars == st->vcap) { + int cap = st->vcap ? st->vcap * PAIR_LEN : CBM_SZ_32; + hvar_t *g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + return false; + } + if (st->nvars) { + memcpy(g, st->vars, (size_t)st->nvars * sizeof(*g)); + } + st->vars = g; + st->vcap = cap; + } + st->vars[st->nvars++] = (hvar_t){name, nl, v}; + return true; +} + +static hvar_t *hst_var(hst_t *st, const hframe_t *fr, const char *name, size_t nl) { + for (int i = st->nvars - SKIP_ONE; i >= fr->var_base; i--) { + if (st->vars[i].nl == nl && memcmp(st->vars[i].name, name, nl) == 0) { + return &st->vars[i]; + } + } + return NULL; +} + +static bool hn_is(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *hn_text(const hframe_t *fr, TSNode n, size_t *len) { + uint32_t b = ts_node_start_byte(n); + *len = ts_node_end_byte(n) - b; + return fr->src + b; +} + +static TSNode hn_field(TSNode n, const char *f) { + return ts_node_child_by_field_name(n, f, (uint32_t)strlen(f)); +} + +/* The name of a variable node ("" for `$`). */ +static const char *hn_var_name(const hframe_t *fr, TSNode var, size_t *nl) { + TSNode nm = hn_field(var, "name"); + if (ts_node_is_null(nm)) { + *nl = 0; + return ""; + } + return hn_text(fr, nm, nl); +} + +/* ── Printing ─────────────────────────────────────────────────────── */ + +/* Go's %v of a basic value into the arena; NULL for one not modelled. */ +static const char *hv_fmt(hst_t *st, const hv_t *v, size_t *len) { + char buf[CBM_SZ_64]; + switch (v->k) { + case HV_STR: + *len = v->sn; + return v->s; + case HV_BOOL: + *len = v->b ? 4 : 5; + return v->b ? "true" : "false"; + case HV_INT: + snprintf(buf, sizeof(buf), "%lld", (long long)v->i); + break; + case HV_FLOAT: + hg_float(v->f, buf, sizeof(buf)); + break; + case HV_NIL: + *len = 5; + return ""; + default: + return NULL; + } + char *out = cbm_arena_strdup(st->a, buf); + *len = out ? strlen(out) : 0; + return out; +} + +/* Sprig's strval. */ +static bool hv_strval(hst_t *st, const hv_t *v, hv_t *out) { + size_t n = 0; + const char *s = hv_fmt(st, v, &n); + if (!s) { + return false; + } + *out = hv_str(s, n); + return true; +} + +/* ── Output and probes ────────────────────────────────────────────── */ + +/* A capture's scalar with its YAML quoting removed; NULL: none / invalid. */ +static const char *hst_scalar(hst_t *st, const char *s, size_t n) { + while (n && (*s == ' ' || *s == '\t')) { + s++; + n--; + } + while (n && (s[n - SKIP_ONE] == ' ' || s[n - SKIP_ONE] == '\t' || s[n - SKIP_ONE] == '\r')) { + n--; + } + if (n == 0) { + return NULL; + } + if (s[0] == '"' || s[0] == '\'') { + /* the closing quote: \" escapes one in "..", '' one in '..' */ + size_t e = SKIP_ONE; + for (; e < n; e++) { + if (s[0] == '"' && s[e] == '\\') { + e++; + } else if (s[0] == '\'' && s[e] == '\'' && e + SKIP_ONE < n && + s[e + SKIP_ONE] == '\'') { + e++; + } else if (s[e] == s[0]) { + break; + } + } + if (e >= n) { + return NULL; + } + for (size_t r = e + SKIP_ONE; r < n; r++) { + if (s[r] == '#') { + break; + } + if (s[r] != ' ' && s[r] != '\t') { + return NULL; + } + } + yyjson_mut_val *v = cbm_helm_values_parse(st->scratch, s, e + SKIP_ONE); + return v && yyjson_mut_is_str(v) ? cbm_arena_strdup(st->a, yyjson_mut_get_str(v)) : NULL; + } + for (size_t i = 0; i < n; i++) { + if (s[i] == '#' && i > 0 && (s[i - SKIP_ONE] == ' ' || s[i - SKIP_ONE] == '\t')) { + n = i; + break; + } + if (s[i] == ':' && (i + SKIP_ONE == n || s[i + SKIP_ONE] == ' ')) { + return NULL; /* a mapping, no scalar */ + } + } + while (n && (s[n - SKIP_ONE] == ' ' || s[n - SKIP_ONE] == '\t')) { + n--; + } + return n ? cbm_arena_strndup(st->a, s, n) : NULL; +} + +static void hst_capture_end(hst_t *st) { + if (st->capture < 0) { + return; + } + cbm_helm_probe_t *p = &st->probes[st->capture]; + const char **img = cbm_arena_alloc(st->a, (size_t)(p->n + SKIP_ONE) * sizeof(*img)); + const char **keys = cbm_arena_alloc(st->a, (size_t)(p->n + SKIP_ONE) * sizeof(*keys)); + if (img && keys) { + if (p->n) { + memcpy(img, p->image, (size_t)p->n * sizeof(*img)); + memcpy(keys, p->keys, (size_t)p->n * sizeof(*keys)); + } + img[p->n] = st->cap.unk ? NULL : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); + /* the leaves read: a path that is the prefix of another is dropped */ + const char *json = NULL; + for (int i = 0; i < st->nck; i++) { + size_t li = strlen(st->ckeys[i]); + bool prefix = false; + for (int j = 0; j < st->nck && !prefix; j++) { + prefix = j != i && strncmp(st->ckeys[j], st->ckeys[i], li) == 0 && + st->ckeys[j][li] == '.'; + } + if (prefix || !st->ckeys[i][0]) { + continue; + } + char esc[CBM_SZ_1K]; + size_t o = 0; + for (const char *c = st->ckeys[i]; *c && o + 3 < sizeof(esc); c++) { + if (*c == '"' || *c == '\\') { + esc[o++] = '\\'; + } + esc[o++] = *c; + } + esc[o] = '\0'; + json = cbm_arena_sprintf(st->a, "%s%s\"%s\"", json ? json : "[", json ? "," : "", esc); + } + keys[p->n] = json ? cbm_arena_sprintf(st->a, "%s]", json) : NULL; + p->image = img; + p->keys = keys; + p->n++; + } + st->capture = -SKIP_ONE; + st->cap = (hsink_t){0}; + st->nck = 0; +} + +/* Output of the top-level template: into the capture until its newline. */ +static void hst_out(hst_t *st, hframe_t *fr, const char *p, size_t n) { + if (fr->out) { + hs_append(st, fr->out, p, n); + return; + } + if (st->capture < 0) { + return; + } + const char *nl = memchr(p, '\n', n); + hs_append(st, &st->cap, p, nl ? (size_t)(nl - p) : n); + if (nl) { + hst_capture_end(st); + } +} + +static void hst_out_unknown(hst_t *st, hframe_t *fr) { + if (fr->out) { + fr->out->unk = true; + } else if (st->capture >= 0) { + st->cap.unk = true; + } +} + +/* Output of a value as text/template prints it (nil: "", as Helm). */ +static void hst_out_value(hst_t *st, hframe_t *fr, const hv_t *v) { + if (v->k == HV_NIL) { + return; + } + size_t n = 0; + const char *s = hv_fmt(st, v, &n); + if (!s) { + hst_out_unknown(st, fr); + return; + } + hst_out(st, fr, s, n); +} + +static bool hn_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; +} + +/* A text node, trimmed by an adjacent `-}}` / `{{-`; at the top level each + * probe whose `image:` key it holds starts its capture after the key. */ +static void hst_text(hst_t *st, hframe_t *fr, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + /* the markers are lexical -- `{{- ` / ` -}}` beside the text, whichever + * action (of whatever nesting) they open or close */ + if (b >= 4 && strncmp(fr->src + b - 3, "-}}", 3) == 0 && hn_ws(fr->src[b - 4])) { + while (b < e && hn_ws(fr->src[b])) { + b++; + } + } + if (strncmp(fr->src + e, "{{-", 3) == 0 && hn_ws(fr->src[e + 3])) { + while (e > b && hn_ws(fr->src[e - SKIP_ONE])) { + e--; + } + } + if (fr->out || fr->src != st->top_src) { + hst_out(st, fr, fr->src + b, e - b); + return; + } + uint32_t at = b; + for (;;) { + /* the next probe key in [at, e) */ + uint32_t key = e; + int which = -SKIP_ONE; + for (int i = 0; i < st->nprobes; i++) { + int line = st->probes[i].line; + if (line < SKIP_ONE || line > st->nlines) { + continue; + } + uint32_t k = st->lines[line - SKIP_ONE]; + while (k < st->top_len && (st->top_src[k] == ' ' || st->top_src[k] == '\t')) { + k++; + } + if (k < st->top_len && st->top_src[k] == '-' && k + SKIP_ONE < st->top_len && + st->top_src[k + SKIP_ONE] == ' ') { + k++; + while (k < st->top_len && st->top_src[k] == ' ') { + k++; + } + } + if (k >= at && k < key && k + strlen("image:") <= e && + memcmp(st->top_src + k, "image:", strlen("image:")) == 0) { + key = k; + which = i; + } + } + if (which < 0) { + hst_out(st, fr, fr->src + at, e - at); + return; + } + uint32_t after = key + (uint32_t)strlen("image:"); + hst_out(st, fr, fr->src + at, after - at); + hst_capture_end(st); + st->capture = which; + st->cap = (hsink_t){0}; + st->nck = 0; + at = after; + } +} + +/* ── Evaluation ───────────────────────────────────────────────────── */ + +static hv_t he_eval(hst_t *st, hframe_t *fr, TSNode n); +static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field); +static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *into); + +/* Field `name` of value v. */ +static hv_t he_field(hst_t *st, const hv_t *v, const char *name, size_t nl) { + if (v->k == HV_MAP) { + for (int i = 0; i < v->m->n; i++) { + if (strlen(v->m->keys[i]) == nl && memcmp(v->m->keys[i], name, nl) == 0) { + hst_record(st, v->m->vals[i].path); + return v->m->vals[i]; + } + } + return v->m->strict ? HV_UNKNOWN : hv_nil(); + } + if (v->k == HV_JSON && yyjson_mut_is_obj(v->j)) { + const char *path = hst_path(st, v->path, name, nl); + yyjson_mut_val *c = yyjson_mut_obj_getn(v->j, name, nl); + hst_record(st, path); + return hv_json(c, path); + } + return HV_UNKNOWN; /* a field of nil, of a scalar, of a list: an error */ +} + +static hv_t he_index1(hst_t *st, const hv_t *v, const hv_t *k) { + if (hv_is_map(v)) { + if (k->k != HV_STR) { + return HV_UNKNOWN; + } + return he_field(st, v, k->s, k->sn); + } + if (hv_is_list(v)) { + if (k->k != HV_INT) { + return HV_UNKNOWN; + } + if (v->k == HV_LIST) { + return k->i >= 0 && k->i < v->l->n ? v->l->items[k->i] : HV_UNKNOWN; + } + yyjson_mut_val *c = k->i >= 0 ? yyjson_mut_arr_get(v->j, (size_t)k->i) : NULL; + if (!c) { + return HV_UNKNOWN; + } + const char *path = + v->path ? cbm_arena_sprintf(st->a, "%s[%lld]", v->path, (long long)k->i) : NULL; + hst_record(st, path); + return hv_json(c, path); + } + return v->k == HV_NIL ? hv_nil() : HV_UNKNOWN; +} + +/* Sprig empty. */ +static bool hv_empty(const hv_t *v) { + return !hv_truth(v); +} + +static bool hv_eq(const hv_t *a, const hv_t *b, bool *ok) { + *ok = true; + if (a->k == HV_STR && b->k == HV_STR) { + return a->sn == b->sn && memcmp(a->s, b->s, a->sn) == 0; + } + if (a->k == HV_BOOL && b->k == HV_BOOL) { + return a->b == b->b; + } + if (a->k == HV_INT && b->k == HV_INT) { + return a->i == b->i; + } + if (a->k == HV_FLOAT && b->k == HV_FLOAT) { + return a->f == b->f; + } + *ok = false; /* incompatible kinds: an error in Go */ + return false; +} + +static hv_t hv_mklist(hst_t *st, const hv_t *items, int n) { + hv_list_t *l = cbm_arena_alloc(st->a, sizeof(*l)); + hv_t *v = n ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*v)) : NULL; + if (!l || (n && !v)) { + return HV_UNKNOWN; + } + if (n) { + memcpy(v, items, (size_t)n * sizeof(*v)); + } + *l = (hv_list_t){v, n}; + return (hv_t){.k = HV_LIST, .l = l}; +} + +/* The items of a list value into an arena array; -1 when it is none. */ +static int hv_items(hst_t *st, const hv_t *v, hv_t **out) { + if (v->k == HV_LIST) { + *out = v->l->items; + return v->l->n; + } + if (v->k == HV_JSON && yyjson_mut_is_arr(v->j)) { + int n = (int)yyjson_mut_arr_size(v->j); + hv_t *items = n ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*items)) : NULL; + if (n && !items) { + return -SKIP_ONE; + } + size_t idx, max; + yyjson_mut_val *it; + yyjson_mut_arr_foreach(v->j, idx, max, it) { + const char *path = v->path ? cbm_arena_sprintf(st->a, "%s[%zu]", v->path, idx) : NULL; + items[idx] = hv_json(it, path); + } + *out = items; + return n; + } + if (v->k == HV_NIL) { + *out = NULL; + return 0; + } + return -SKIP_ONE; +} + +static int he_key_cmp(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* A map's keys, sorted (Go's order for range), into an arena array. */ +static int hv_keys(hst_t *st, const hv_t *v, const char ***out) { + int n = hv_len(v); + const char **keys = n > 0 ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*keys)) : NULL; + if (n > 0 && !keys) { + return -SKIP_ONE; + } + if (v->k == HV_MAP) { + for (int i = 0; i < n; i++) { + keys[i] = v->m->keys[i]; + } + } else { + size_t idx, max; + yyjson_mut_val *k, *val; + yyjson_mut_obj_foreach(v->j, idx, max, k, val) { + (void)val; + keys[idx] = yyjson_mut_get_str(k); + } + } + if (n > SKIP_ONE) { + qsort(keys, (size_t)n, sizeof(*keys), he_key_cmp); + } + *out = keys; + return n; +} + +static bool hv_to_int(const hv_t *v, int64_t *out) { + switch (v->k) { + case HV_INT: + *out = v->i; + return true; + case HV_FLOAT: + *out = (int64_t)v->f; + return true; + case HV_BOOL: + *out = v->b ? SKIP_ONE : 0; + return true; + case HV_NIL: + *out = 0; + return true; + case HV_STR: { + char buf[CBM_SZ_64]; + if (v->sn == 0 || v->sn >= sizeof(buf)) { + *out = 0; + return v->sn == 0; + } + memcpy(buf, v->s, v->sn); + buf[v->sn] = '\0'; + char *end = NULL; + long long i = strtoll(buf, &end, 0); + if (end && !*end) { + *out = i; + return true; + } + double d = strtod(buf, &end); + *out = end && !*end ? (int64_t)d : 0; + return true; + } + default: + return false; + } +} + +/* Go's %q of s. */ +static const char *hg_quote(hst_t *st, const char *s, size_t n, char q) { + char *out = cbm_arena_alloc(st->a, n * 4 + 3); + if (!out) { + return NULL; + } + size_t k = 0; + out[k++] = q; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (q == '"' && (c == '"' || c == '\\')) { + out[k++] = '\\'; + out[k++] = (char)c; + } else if (q == '"' && c < 0x20) { + static const char HEX[] = "0123456789abcdef"; + const char *named = c == '\n' ? "\\n" : c == '\t' ? "\\t" : c == '\r' ? "\\r" : NULL; + if (named) { + out[k++] = named[0]; + out[k++] = named[1]; + } else { + out[k++] = '\\'; + out[k++] = 'x'; + out[k++] = HEX[c >> 4]; + out[k++] = HEX[c & 0xF]; + } + } else { + out[k++] = (char)c; + } + } + out[k++] = q; + out[k] = '\0'; + return out; +} + +/* fmt.Sprintf for the verbs %s %v %d %q %% without flags; NULL otherwise. */ +static const char *he_printf(hst_t *st, const hv_t *args, int n) { + if (n < SKIP_ONE || args[0].k != HV_STR) { + return NULL; + } + hsink_t out = {0}; + int ai = SKIP_ONE; + const char *f = args[0].s; + size_t fl = args[0].sn; + for (size_t i = 0; i < fl; i++) { + if (f[i] != '%') { + hs_append(st, &out, f + i, SKIP_ONE); + continue; + } + if (++i >= fl) { + return NULL; + } + char verb = f[i]; + if (verb == '%') { + hs_append(st, &out, "%", SKIP_ONE); + continue; + } + if (ai >= n) { + return NULL; + } + const hv_t *a = &args[ai++]; + size_t sl = 0; + const char *s = NULL; + if (verb == 's' && a->k == HV_STR) { + s = a->s; + sl = a->sn; + } else if (verb == 'v') { + s = hv_fmt(st, a, &sl); + } else if (verb == 'd' && a->k == HV_INT) { + s = hv_fmt(st, a, &sl); + } else if (verb == 'q' && a->k == HV_STR) { + s = hg_quote(st, a->s, a->sn, '"'); + sl = s ? strlen(s) : 0; + } + if (!s) { + return NULL; /* %!s(float64=...) and the like: no image */ + } + hs_append(st, &out, s, sl); + } + if (ai != n || out.unk) { + return NULL; + } + return hs_str(st, &out); +} + +static hv_t he_strfn(hst_t *st, const char *fn, const hv_t *a, int n) { + /* Sprig string functions take the subject last. */ + hv_t s; + if (n < SKIP_ONE || !hv_strval(st, &a[n - SKIP_ONE], &s)) { + return HV_UNKNOWN; + } + if (strcmp(fn, "trim") == 0 && n == SKIP_ONE) { + size_t b = 0; + size_t e = s.sn; + while (b < e && hn_ws(s.s[b])) { + b++; + } + while (e > b && hn_ws(s.s[e - SKIP_ONE])) { + e--; + } + return hv_str(s.s + b, e - b); + } + if ((strcmp(fn, "lower") == 0 || strcmp(fn, "upper") == 0) && n == SKIP_ONE) { + char *o = cbm_arena_strndup(st->a, s.s, s.sn); + for (size_t i = 0; o && i < s.sn; i++) { + o[i] = + (char)(fn[0] == 'l' ? tolower((unsigned char)o[i]) : toupper((unsigned char)o[i])); + } + return o ? hv_str(o, s.sn) : HV_UNKNOWN; + } + if (n != PAIR_LEN && strcmp(fn, "replace") != 0) { + return HV_UNKNOWN; + } + hv_t p; + if (!hv_strval(st, &a[0], &p)) { + return HV_UNKNOWN; + } + if (strcmp(fn, "trimPrefix") == 0) { + bool has = p.sn <= s.sn && memcmp(s.s, p.s, p.sn) == 0; + return has ? hv_str(s.s + p.sn, s.sn - p.sn) : s; + } + if (strcmp(fn, "trimSuffix") == 0) { + bool has = p.sn <= s.sn && memcmp(s.s + s.sn - p.sn, p.s, p.sn) == 0; + return has ? hv_str(s.s, s.sn - p.sn) : s; + } + if (strcmp(fn, "trimAll") == 0) { + size_t b = 0; + size_t e = s.sn; + while (b < e && memchr(p.s, s.s[b], p.sn)) { + b++; + } + while (e > b && memchr(p.s, s.s[e - SKIP_ONE], p.sn)) { + e--; + } + return hv_str(s.s + b, e - b); + } + if (strcmp(fn, "contains") == 0 || strcmp(fn, "hasPrefix") == 0 || + strcmp(fn, "hasSuffix") == 0) { + if (p.sn > s.sn) { + return hv_bool(false); + } + if (fn[0] == 'c') { + char *hs = cbm_arena_strndup(st->a, s.s, s.sn); + char *ns = cbm_arena_strndup(st->a, p.s, p.sn); + return hs && ns ? hv_bool(strstr(hs, ns) != NULL) : HV_UNKNOWN; + } + const char *at = fn[3] == 'P' ? s.s : s.s + s.sn - p.sn; + return hv_bool(memcmp(at, p.s, p.sn) == 0); + } + if (strcmp(fn, "trunc") == 0) { + int64_t c = 0; + if (!hv_to_int(&a[0], &c)) { + return HV_UNKNOWN; + } + if (c >= 0) { + return hv_str(s.s, (size_t)c < s.sn ? (size_t)c : s.sn); + } + size_t keep = (size_t)(-c) < s.sn ? (size_t)(-c) : s.sn; + return hv_str(s.s + s.sn - keep, keep); + } + if (strcmp(fn, "replace") == 0 && n == 3) { + hv_t to; + if (!hv_strval(st, &a[SKIP_ONE], &to) || p.sn == 0) { + return HV_UNKNOWN; + } + hsink_t out = {0}; + for (size_t i = 0; i < s.sn;) { + if (i + p.sn <= s.sn && memcmp(s.s + i, p.s, p.sn) == 0) { + hs_append(st, &out, to.s, to.sn); + i += p.sn; + } else { + hs_append(st, &out, s.s + i, SKIP_ONE); + i++; + } + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + return HV_UNKNOWN; +} + +/* A function call `fn` with its evaluated arguments (the piped value last). */ +static hv_t he_apply(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { + for (int i = 0; i < n; i++) { + if (a[i].k == HV_UNK) { + return HV_UNKNOWN; + } + } + if (strcmp(fn, "include") == 0 && n == PAIR_LEN && a[0].k == HV_STR) { + hsink_t out = {0}; + char *name = cbm_arena_strndup(st->a, a[0].s, a[0].sn); + hv_t r = name ? he_render_define(st, name, a[1], &out) : HV_UNKNOWN; + return r.k == HV_UNK || out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "tpl") == 0 && n == PAIR_LEN && a[0].k == HV_STR) { + if (!st->parser) { + return HV_UNKNOWN; + } + char *src = cbm_arena_strndup(st->a, a[0].s, a[0].sn); + TSTree *t = src ? ts_parser_parse_string(st->parser, NULL, src, (uint32_t)a[0].sn) : NULL; + if (!t) { + return HV_UNKNOWN; + } + if (st->ntrees == st->tcap) { + int cap = st->tcap ? st->tcap * PAIR_LEN : CBM_SZ_16; + TSTree **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + ts_tree_delete(t); + return HV_UNKNOWN; + } + if (st->ntrees) { + memcpy(g, st->trees, (size_t)st->ntrees * sizeof(*g)); + } + st->trees = g; + st->tcap = cap; + } + st->trees[st->ntrees++] = t; + TSNode root = ts_tree_root_node(t); + if (ts_node_has_error(root) || ++st->nesting > HELM_MAX_NESTING) { + st->nesting -= ts_node_has_error(root) ? 0 : SKIP_ONE; + return HV_UNKNOWN; + } + hsink_t out = {0}; + int mark = st->nvars; + hframe_t sub = {.src = src, .dot = a[1], .var_base = st->nvars, .out = &out}; + hst_var_push(st, "", 0, a[1]); + he_items(st, &sub, root, NULL); + st->nvars = mark; + st->nesting--; + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "default") == 0 && n >= SKIP_ONE) { + return n == SKIP_ONE || hv_empty(&a[SKIP_ONE]) ? a[0] : a[SKIP_ONE]; + } + if (strcmp(fn, "coalesce") == 0) { + for (int i = 0; i < n; i++) { + if (!hv_empty(&a[i])) { + return a[i]; + } + } + return hv_nil(); + } + if (strcmp(fn, "ternary") == 0 && n == 3) { + return a[2].k == HV_BOOL ? (a[2].b ? a[0] : a[SKIP_ONE]) : HV_UNKNOWN; + } + if (strcmp(fn, "empty") == 0 && n == SKIP_ONE) { + return hv_bool(hv_empty(&a[0])); + } + if (strcmp(fn, "not") == 0 && n == SKIP_ONE) { + return hv_bool(!hv_truth(&a[0])); + } + if ((strcmp(fn, "eq") == 0 && n >= PAIR_LEN) || (strcmp(fn, "ne") == 0 && n == PAIR_LEN)) { + bool any = false; + for (int i = SKIP_ONE; i < n; i++) { + bool ok = false; + any = any || hv_eq(&a[0], &a[i], &ok); + if (!ok) { + return HV_UNKNOWN; + } + } + return hv_bool(fn[0] == 'e' ? any : !any); + } + if (strcmp(fn, "required") == 0 && n == PAIR_LEN) { + bool missing = a[1].k == HV_NIL || (a[1].k == HV_STR && a[1].sn == 0); + return missing ? HV_UNKNOWN : a[1]; + } + if (strcmp(fn, "printf") == 0) { + const char *s = he_printf(st, a, n); + return s ? hv_str(s, strlen(s)) : HV_UNKNOWN; + } + if (strcmp(fn, "print") == 0 || strcmp(fn, "cat") == 0) { + hsink_t out = {0}; + bool prev_str = true; + int written = 0; + for (int i = 0; i < n; i++) { + if (fn[0] == 'c' && a[i].k == HV_NIL) { + continue; + } + size_t sl = 0; + const char *s = hv_fmt(st, &a[i], &sl); + if (!s) { + return HV_UNKNOWN; + } + bool is_str = a[i].k == HV_STR; + /* fmt.Sprint spaces two operands neither of which is a string; cat always */ + if (written && (fn[0] == 'c' || (!is_str && !prev_str))) { + hs_append(st, &out, " ", SKIP_ONE); + } + hs_append(st, &out, s, sl); + prev_str = is_str; + written++; + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "quote") == 0 || strcmp(fn, "squote") == 0) { + hsink_t out = {0}; + int written = 0; + for (int i = 0; i < n; i++) { + hv_t s; + if (a[i].k == HV_NIL) { + continue; + } + if (!hv_strval(st, &a[i], &s)) { + return HV_UNKNOWN; + } + const char *q = fn[0] == 'q' ? hg_quote(st, s.s, s.sn, '"') + : cbm_arena_sprintf(st->a, "'%.*s'", (int)s.sn, s.s); + if (!q) { + return HV_UNKNOWN; + } + if (written++) { + hs_append(st, &out, " ", SKIP_ONE); + } + hs_append(st, &out, q, strlen(q)); + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "toString") == 0 && n == SKIP_ONE) { + hv_t s; + return hv_strval(st, &a[0], &s) ? s : HV_UNKNOWN; + } + if ((strcmp(fn, "int") == 0 || strcmp(fn, "int64") == 0 || strcmp(fn, "toInt") == 0) && + n == SKIP_ONE) { + int64_t i = 0; + return hv_to_int(&a[0], &i) ? hv_int(i) : HV_UNKNOWN; + } + if (strcmp(fn, "float64") == 0 && n == SKIP_ONE) { + int64_t i = 0; + return a[0].k == HV_FLOAT ? a[0] : hv_to_int(&a[0], &i) ? hv_float((double)i) : HV_UNKNOWN; + } + if (strcmp(fn, "len") == 0 && n == SKIP_ONE) { + int l = hv_len(&a[0]); + return l >= 0 ? hv_int(l) : HV_UNKNOWN; + } + if (strcmp(fn, "dict") == 0) { + hv_map_t *m = cbm_arena_calloc(st->a, sizeof(*m)); + int pairs = (n + SKIP_ONE) / PAIR_LEN; + if (!m) { + return HV_UNKNOWN; + } + m->keys = pairs ? cbm_arena_alloc(st->a, (size_t)pairs * sizeof(*m->keys)) : NULL; + m->vals = pairs ? cbm_arena_alloc(st->a, (size_t)pairs * sizeof(*m->vals)) : NULL; + if (pairs && (!m->keys || !m->vals)) { + return HV_UNKNOWN; + } + for (int i = 0; i + SKIP_ONE <= n; i += PAIR_LEN) { + hv_t k; + if (!hv_strval(st, &a[i], &k)) { + return HV_UNKNOWN; + } + const char *key = cbm_arena_strndup(st->a, k.s, k.sn); + hv_t val = i + SKIP_ONE < n ? a[i + SKIP_ONE] : hv_str("", 0); + int at = m->n; + for (int j = 0; j < m->n; j++) { + if (strcmp(m->keys[j], key) == 0) { + at = j; + } + } + m->keys[at] = key; + m->vals[at] = val; + m->n += at == m->n; + } + return (hv_t){.k = HV_MAP, .m = m}; + } + if (strcmp(fn, "list") == 0) { + return hv_mklist(st, a, n); + } + if (strcmp(fn, "get") == 0 && n == PAIR_LEN) { + if (!hv_is_map(&a[0]) || a[1].k != HV_STR) { + return HV_UNKNOWN; + } + hv_t v = he_field(st, &a[0], a[1].s, a[1].sn); + return v.k == HV_NIL ? hv_str("", 0) : v; + } + if (strcmp(fn, "hasKey") == 0 && n == PAIR_LEN) { + if (!hv_is_map(&a[0]) || a[1].k != HV_STR) { + return HV_UNKNOWN; + } + if (a[0].k == HV_MAP) { + for (int i = 0; i < a[0].m->n; i++) { + if (strlen(a[0].m->keys[i]) == a[1].sn && + memcmp(a[0].m->keys[i], a[1].s, a[1].sn) == 0) { + return hv_bool(true); + } + } + return hv_bool(false); + } + hst_record(st, hst_path(st, a[0].path, a[1].s, a[1].sn)); + return hv_bool(yyjson_mut_obj_getn(a[0].j, a[1].s, a[1].sn) != NULL); + } + if (strcmp(fn, "index") == 0 && n >= SKIP_ONE) { + hv_t v = a[0]; + for (int i = SKIP_ONE; i < n && v.k != HV_UNK; i++) { + v = he_index1(st, &v, &a[i]); + } + return v; + } + if ((strcmp(fn, "first") == 0 || strcmp(fn, "last") == 0) && n == SKIP_ONE) { + hv_t *items = NULL; + int c = hv_items(st, &a[0], &items); + if (c < 0) { + return HV_UNKNOWN; + } + return c == 0 ? hv_nil() : items[fn[0] == 'f' ? 0 : c - SKIP_ONE]; + } + if (strcmp(fn, "join") == 0 && n == PAIR_LEN) { + hv_t *items = NULL; + int c = hv_items(st, &a[1], &items); + hv_t sep; + if (c < 0 || !hv_strval(st, &a[0], &sep)) { + return HV_UNKNOWN; + } + hsink_t out = {0}; + for (int i = 0; i < c; i++) { + hv_t s; + if (!hv_strval(st, &items[i], &s)) { + return HV_UNKNOWN; + } + if (i) { + hs_append(st, &out, sep.s, sep.sn); + } + hs_append(st, &out, s.s, s.sn); + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if (strcmp(fn, "splitList") == 0 && n == PAIR_LEN && a[0].k == HV_STR && a[1].k == HV_STR && + a[0].sn > 0) { + int cap = SKIP_ONE; + for (size_t i = 0; i + a[0].sn <= a[1].sn; i++) { + cap += memcmp(a[1].s + i, a[0].s, a[0].sn) == 0; + } + hv_t *items = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*items)); + if (!items) { + return HV_UNKNOWN; + } + int c = 0; + size_t from = 0; + for (size_t i = 0; i + a[0].sn <= a[1].sn;) { + if (memcmp(a[1].s + i, a[0].s, a[0].sn) == 0) { + items[c++] = hv_str(a[1].s + from, i - from); + i += a[0].sn; + from = i; + } else { + i++; + } + } + items[c++] = hv_str(a[1].s + from, a[1].sn - from); + return hv_mklist(st, items, c); + } + if ((strcmp(fn, "nindent") == 0 || strcmp(fn, "indent") == 0) && n == PAIR_LEN) { + int64_t w = 0; + hv_t s; + if (!hv_to_int(&a[0], &w) || w < 0 || !hv_strval(st, &a[1], &s)) { + return HV_UNKNOWN; + } + hsink_t out = {0}; + if (fn[0] == 'n') { + hs_append(st, &out, "\n", SKIP_ONE); + } + for (int64_t i = 0; i < w; i++) { + hs_append(st, &out, " ", SKIP_ONE); + } + for (size_t i = 0; i < s.sn; i++) { + hs_append(st, &out, s.s + i, SKIP_ONE); + for (int64_t k = 0; s.s[i] == '\n' && k < w; k++) { + hs_append(st, &out, " ", SKIP_ONE); + } + } + return out.unk ? HV_UNKNOWN : hv_str(hs_str(st, &out), out.n); + } + if ((strcmp(fn, "kindIs") == 0 && n == PAIR_LEN) || (strcmp(fn, "kindOf") == 0 && n == 1)) { + const hv_t *v = &a[n - SKIP_ONE]; + const char *kind = v->k == HV_NIL ? "invalid" + : v->k == HV_BOOL ? "bool" + : v->k == HV_INT ? (v->path ? "int64" : "int") + : v->k == HV_FLOAT ? "float64" + : v->k == HV_STR ? "string" + : hv_is_map(v) ? "map" + : hv_is_list(v) ? "slice" + : NULL; + if (!kind) { + return HV_UNKNOWN; + } + if (fn[4] == 'O') { + return hv_str(kind, strlen(kind)); + } + return a[0].k == HV_STR + ? hv_bool(strlen(kind) == a[0].sn && memcmp(kind, a[0].s, a[0].sn) == 0) + : HV_UNKNOWN; + } + if (strcmp(fn, "dig") == 0 && n >= 3) { + hv_t v = a[n - SKIP_ONE]; + for (int i = 0; i < n - PAIR_LEN; i++) { + if (!hv_is_map(&v) || a[i].k != HV_STR) { + return HV_UNKNOWN; + } + v = he_field(st, &v, a[i].s, a[i].sn); + if (v.k == HV_NIL) { + return a[n - PAIR_LEN]; + } + } + return v; + } + if (strcmp(fn, "trim") == 0 || strcmp(fn, "trimPrefix") == 0 || strcmp(fn, "trimSuffix") == 0 || + strcmp(fn, "trimAll") == 0 || strcmp(fn, "lower") == 0 || strcmp(fn, "upper") == 0 || + strcmp(fn, "replace") == 0 || strcmp(fn, "trunc") == 0 || strcmp(fn, "contains") == 0 || + strcmp(fn, "hasPrefix") == 0 || strcmp(fn, "hasSuffix") == 0) { + return he_strfn(st, fn, a, n); + } + (void)fr; + return HV_UNKNOWN; /* not modelled (toYaml, lookup, regex, semver, merge, ...) */ +} + +/* A function call node -- `fn args...` or a bare `fn` -- with `piped` (may + * be NULL) as its last argument. and / or evaluate lazily. */ +static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { + TSNode fnode = hn_is(call, "identifier") ? call : hn_field(call, "function"); + if (ts_node_is_null(fnode)) { + return HV_UNKNOWN; + } + size_t fl = 0; + const char *ft = hn_text(fr, fnode, &fl); + char fn[CBM_SZ_64]; + if (fl >= sizeof(fn)) { + return HV_UNKNOWN; + } + memcpy(fn, ft, fl); + fn[fl] = '\0'; + TSNode args = hn_is(call, "identifier") ? (TSNode){0} : hn_field(call, "arguments"); + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + bool lazy = strcmp(fn, "and") == 0 || strcmp(fn, "or") == 0; + if (lazy) { + hv_t v = HV_UNKNOWN; + uint32_t total = na + (piped ? SKIP_ONE : 0); + for (uint32_t i = 0; i < total; i++) { + v = i < na ? he_eval(st, fr, ts_node_named_child(args, i)) : *piped; + if (v.k == HV_UNK) { + return HV_UNKNOWN; + } + bool t = hv_truth(&v); + if ((fn[0] == 'a' && !t) || (fn[0] == 'o' && t)) { + return v; + } + } + return total ? v : HV_UNKNOWN; + } + int n = (int)na + (piped ? SKIP_ONE : 0); + hv_t *a = n ? cbm_arena_alloc(st->a, (size_t)n * sizeof(*a)) : NULL; + if (n && !a) { + return HV_UNKNOWN; + } + for (uint32_t i = 0; i < na; i++) { + a[i] = he_eval(st, fr, ts_node_named_child(args, i)); + } + if (piped) { + a[na] = *piped; + } + return he_apply(st, fr, fn, a, n); +} + +static hv_t he_literal_int(hframe_t *fr, TSNode n) { + size_t l = 0; + const char *t = hn_text(fr, n, &l); + char buf[CBM_SZ_64]; + size_t k = 0; + for (size_t i = 0; i < l && k + SKIP_ONE < sizeof(buf); i++) { + if (t[i] != '_') { + buf[k++] = t[i]; + } + } + buf[k] = '\0'; + int base = 10; + const char *p = buf; + if (p[0] == '0' && (p[1] == 'o' || p[1] == 'O')) { + base = 8; + p += PAIR_LEN; + } else if (p[0] == '0' && (p[1] == 'b' || p[1] == 'B')) { + base = 2; + p += PAIR_LEN; + } else if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) { + base = 16; + p += PAIR_LEN; + } else if (p[0] == '0' && p[1]) { + base = 8; + p++; + } + char *end = NULL; + long long v = strtoll(p, &end, base); + return end && !*end ? hv_int(v) : HV_UNKNOWN; +} + +static hv_t he_eval(hst_t *st, hframe_t *fr, TSNode n) { + if (ts_node_is_null(n) || ts_node_has_error(n) || ts_node_is_missing(n)) { + return HV_UNKNOWN; + } + const char *t = ts_node_type(n); + size_t l = 0; + if (strcmp(t, "dot") == 0) { + return fr->dot; + } + if (strcmp(t, "field") == 0) { + TSNode nm = hn_field(n, "name"); + if (ts_node_is_null(nm)) { + return HV_UNKNOWN; + } + const char *name = hn_text(fr, nm, &l); + return he_field(st, &fr->dot, name, l); + } + if (strcmp(t, "variable") == 0) { + const char *name = hn_var_name(fr, n, &l); + hvar_t *v = hst_var(st, fr, name, l); + return v ? v->v : HV_UNKNOWN; + } + if (strcmp(t, "selector_expression") == 0) { + TSNode op = hn_field(n, "operand"); + TSNode fd = hn_field(n, "field"); + if (ts_node_is_null(fd)) { + return HV_UNKNOWN; + } + hv_t base = he_eval(st, fr, op); + if (base.k == HV_UNK) { + return HV_UNKNOWN; + } + const char *name = hn_text(fr, fd, &l); + return he_field(st, &base, name, l); + } + if (strcmp(t, "chained_pipeline") == 0) { + uint32_t c = ts_node_named_child_count(n); + if (c == 0) { + return HV_UNKNOWN; + } + hv_t v = he_eval(st, fr, ts_node_named_child(n, 0)); + for (uint32_t i = SKIP_ONE; i < c && v.k != HV_UNK; i++) { + TSNode stage = ts_node_named_child(n, i); + if (!hn_is(stage, "function_call") && !hn_is(stage, "identifier")) { + return HV_UNKNOWN; + } + v = he_call(st, fr, stage, &v); + } + return v; + } + if (strcmp(t, "function_call") == 0 || strcmp(t, "identifier") == 0) { + return he_call(st, fr, n, NULL); + } + if (strcmp(t, "parenthesized_pipeline") == 0) { + return ts_node_named_child_count(n) == SKIP_ONE ? he_eval(st, fr, ts_node_named_child(n, 0)) + : HV_UNKNOWN; + } + if (strcmp(t, "interpreted_string_literal") == 0 || strcmp(t, "raw_string_literal") == 0) { + const char *s = hn_text(fr, n, &l); + char *out = cbm_arena_alloc(st->a, l * 4 + SKIP_ONE); + size_t k = out ? hg_unquote(s, l, out) : (size_t)-1; + if (k == (size_t)-1) { + return HV_UNKNOWN; + } + out[k] = '\0'; + return hv_str(out, k); + } + if (strcmp(t, "int_literal") == 0) { + return he_literal_int(fr, n); + } + if (strcmp(t, "float_literal") == 0) { + const char *s = hn_text(fr, n, &l); + char *buf = cbm_arena_strndup(st->a, s, l); + return buf ? hv_float(strtod(buf, NULL)) : HV_UNKNOWN; + } + if (strcmp(t, "true") == 0 || strcmp(t, "false") == 0) { + return hv_bool(t[0] == 't'); + } + if (strcmp(t, "nil") == 0) { + return hv_nil(); + } + if (strcmp(t, "variable_definition") == 0 || strcmp(t, "assignment") == 0) { + TSNode var = hn_field(n, "variable"); + hv_t v = he_eval(st, fr, hn_field(n, "value")); + const char *name = ts_node_is_null(var) ? NULL : hn_var_name(fr, var, &l); + if (!name) { + return HV_UNKNOWN; + } + if (t[0] == 'v') { + hst_var_push(st, name, l, v); + } else { + hvar_t *cur = hst_var(st, fr, name, l); + if (!cur) { + return HV_UNKNOWN; + } + cur->v = v; + } + return v; + } + return HV_UNKNOWN; /* method calls, rune / imaginary literals, ... */ +} + +/* ── Execution ────────────────────────────────────────────────────── */ + +static void he_item(hst_t *st, hframe_t *fr, TSNode c); + +/* The named children of `parent` in field `field` (NULL: every one). */ +static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field) { + uint32_t cc = ts_node_child_count(parent); + for (uint32_t i = 0; i < cc && !st->brk && !st->cont; i++) { + TSNode c = ts_node_child(parent, i); + if (!ts_node_is_named(c)) { + continue; + } + const char *f = ts_node_field_name_for_child(parent, i); + if (field ? !(f && strcmp(f, field) == 0) : f != NULL) { + continue; + } + he_item(st, fr, c); + } +} + +/* if / else if / else, and with / else with: the first branch whose + * condition holds runs; an UNKNOWN condition runs none. */ +static void he_branches(hst_t *st, hframe_t *fr, TSNode n, bool with) { + uint32_t cc = ts_node_child_count(n); + bool taken = false; + bool stop = false; + bool active = false; + int mark = st->nvars; + hframe_t sub = *fr; + for (uint32_t i = 0; i < cc && !st->brk && !st->cont; i++) { + TSNode c = ts_node_child(n, i); + if (!ts_node_is_named(c)) { + continue; + } + const char *f = ts_node_field_name_for_child(n, i); + if (!f) { + continue; + } + if (strcmp(f, "condition") == 0) { + if (taken || stop) { + active = false; + continue; + } + hv_t v = he_eval(st, fr, c); + if (v.k == HV_UNK) { + stop = true; + active = false; + hst_out_unknown(st, fr); /* which branch renders is unknown */ + continue; + } + active = hv_truth(&v); + taken = active; + sub = *fr; + if (with && active) { + sub.dot = v; + } + } else if (strcmp(f, "consequence") == 0 || strcmp(f, "option") == 0) { + if (active) { + he_item(st, &sub, c); + } + } else if (strcmp(f, "alternative") == 0) { + if (!taken && !stop) { + he_item(st, fr, c); + } + } + } + st->nvars = mark; +} + +static void he_range(hst_t *st, hframe_t *fr, TSNode n) { + TSNode def = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + if (hn_is(ts_node_named_child(n, i), "range_variable_definition")) { + def = ts_node_named_child(n, i); + } + } + TSNode expr = ts_node_is_null(def) ? hn_field(n, "range") : hn_field(def, "range"); + TSNode ivar = ts_node_is_null(def) ? (TSNode){0} : hn_field(def, "index"); + TSNode evar = ts_node_is_null(def) ? (TSNode){0} : hn_field(def, "element"); + if (!ts_node_is_null(ivar) && ts_node_is_null(evar)) { + evar = ivar; /* range $x := ...: the element */ + ivar = (TSNode){0}; + } + hv_t coll = he_eval(st, fr, expr); + if (coll.k == HV_UNK) { + hst_out_unknown(st, fr); + return; + } + hv_t *items = NULL; + const char **keys = NULL; + int count = 0; + bool is_map = hv_is_map(&coll); + if (is_map) { + count = hv_keys(st, &coll, &keys); + } else if (coll.k == HV_INT) { + count = coll.i > 0 ? (int)coll.i : 0; + } else { + count = hv_items(st, &coll, &items); + } + if (count < 0) { + hst_out_unknown(st, fr); + return; + } + size_t il = 0; + size_t el = 0; + const char *iname = ts_node_is_null(ivar) ? NULL : hn_var_name(fr, ivar, &il); + const char *ename = ts_node_is_null(evar) ? NULL : hn_var_name(fr, evar, &el); + for (int k = 0; k < count; k++) { + int mark = st->nvars; + hv_t elem; + hv_t index; + if (is_map) { + elem = he_field(st, &coll, keys[k], strlen(keys[k])); + index = hv_str(keys[k], strlen(keys[k])); + } else if (coll.k == HV_INT) { + elem = hv_int(k); + index = hv_int(k); + } else { + elem = items[k]; + index = hv_int(k); + } + if (iname) { + hst_var_push(st, iname, il, index); + } + if (ename) { + hst_var_push(st, ename, el, elem); + } + hframe_t sub = *fr; + sub.dot = elem; + he_items(st, &sub, n, "body"); + st->nvars = mark; + st->cont = false; + if (st->brk) { + st->brk = false; + break; + } + } + if (count == 0) { + he_items(st, fr, n, "alternative"); + } +} + +/* `template "name" arg` / a block: the define rendered into the output. */ +static void he_template(hst_t *st, hframe_t *fr, TSNode n) { + TSNode nm = hn_field(n, "name"); + TSNode arg = hn_field(n, "argument"); + size_t l = 0; + const char *s = ts_node_is_null(nm) ? NULL : hn_text(fr, nm, &l); + char *name = s ? cbm_arena_alloc(st->a, l * 4 + SKIP_ONE) : NULL; + size_t k = name ? hg_unquote(s, l, name) : (size_t)-1; + if (k == (size_t)-1) { + hst_out_unknown(st, fr); + return; + } + name[k] = '\0'; + hv_t a = ts_node_is_null(arg) ? hv_nil() : he_eval(st, fr, arg); + if (a.k == HV_UNK) { + hst_out_unknown(st, fr); + return; + } + if (fr->out) { + if (he_render_define(st, name, a, fr->out).k == HV_UNK) { + fr->out->unk = true; + } + return; + } + /* at the top level the define's text is output that probes may read */ + hsink_t out = {0}; + if (he_render_define(st, name, a, &out).k == HV_UNK || out.unk) { + hst_out_unknown(st, fr); + return; + } + hst_out(st, fr, out.buf ? out.buf : "", out.n); +} + +static void he_item(hst_t *st, hframe_t *fr, TSNode c) { + const char *t = ts_node_type(c); + if (strcmp(t, "text") == 0) { + hst_text(st, fr, c); + } else if (strcmp(t, "comment") == 0 || strcmp(t, "define_action") == 0) { + return; + } else if (strcmp(t, "if_action") == 0) { + he_branches(st, fr, c, false); + } else if (strcmp(t, "with_action") == 0) { + he_branches(st, fr, c, true); + } else if (strcmp(t, "range_action") == 0) { + he_range(st, fr, c); + } else if (strcmp(t, "template_action") == 0 || strcmp(t, "block_action") == 0) { + he_template(st, fr, c); + } else if (strcmp(t, "break_action") == 0) { + st->brk = true; + } else if (strcmp(t, "continue_action") == 0) { + st->cont = true; + } else if (strcmp(t, "variable_definition") == 0 || strcmp(t, "assignment") == 0) { + (void)he_eval(st, fr, c); + } else { + hv_t v = he_eval(st, fr, c); + if (v.k == HV_UNK) { + hst_out_unknown(st, fr); + } else { + hst_out_value(st, fr, &v); + } + } +} + +/* Define `name` rendered with dot / $ = arg into `into`. */ +static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *into) { + const hd_def_t *d = hd_find(st->defs, name); + if (!d || ++st->nesting > HELM_MAX_NESTING) { + st->nesting -= d ? SKIP_ONE : 0; + return HV_UNKNOWN; + } + int mark = st->nvars; + hframe_t sub = {.src = d->file->src, .dot = arg, .var_base = st->nvars, .out = into}; + hst_var_push(st, "", 0, arg); + bool brk = st->brk; + bool cont = st->cont; + st->brk = false; + st->cont = false; + he_items(st, &sub, d->node, "body"); + st->brk = brk; + st->cont = cont; + st->nvars = mark; + st->nesting--; + return into->unk ? HV_UNKNOWN : hv_nil(); +} + +/* ── Entry ────────────────────────────────────────────────────────── */ + +static hv_map_t *hv_struct(hst_t *st, int n) { + hv_map_t *m = cbm_arena_calloc(st->a, sizeof(*m)); + if (m) { + m->keys = cbm_arena_alloc(st->a, (size_t)n * sizeof(*m->keys)); + m->vals = cbm_arena_alloc(st->a, (size_t)n * sizeof(*m->vals)); + m->strict = true; + if (!m->keys || !m->vals) { + return NULL; + } + } + return m; +} + +static void hv_struct_set(hv_map_t *m, const char *k, hv_t v) { + m->keys[m->n] = k; + m->vals[m->n++] = v; +} + +static hv_t hv_cstr(const char *s) { + return s ? hv_str(s, strlen(s)) : hv_nil(); +} + +void cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, + yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, + const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes) { + (void)rel; + for (int i = 0; i < nprobes; i++) { + probes[i].n = 0; + probes[i].image = NULL; + probes[i].keys = NULL; + } + if (!src || nprobes == 0) { + return; + } + hst_t st = {.a = a, + .defs = defs, + .top_src = src, + .top_len = len, + .probes = probes, + .nprobes = nprobes, + .capture = -SKIP_ONE}; + st.scratch = yyjson_mut_doc_new(cbm_json_writer_allocator()); + st.parser = ts_parser_new(); + if (!st.scratch || !st.parser || + !ts_parser_set_language(st.parser, cbm_ts_language(CBM_LANG_GOTEMPLATE))) { + if (st.parser) { + ts_parser_delete(st.parser); + } + if (st.scratch) { + yyjson_mut_doc_free(st.scratch); + } + return; + } + int nl = SKIP_ONE; + for (size_t i = 0; i < len; i++) { + nl += src[i] == '\n'; + } + uint32_t *lines = cbm_arena_alloc(a, (size_t)nl * sizeof(*lines)); + TSTree *tree = lines ? ts_parser_parse_string(st.parser, NULL, src, (uint32_t)len) : NULL; + if (tree) { + int k = 0; + lines[k++] = 0; + for (size_t i = 0; i < len; i++) { + if (src[i] == '\n' && k < nl) { + lines[k++] = (uint32_t)(i + SKIP_ONE); + } + } + st.lines = lines; + st.nlines = k; + yyjson_mut_val *vals = values ? values : yyjson_mut_obj(st.scratch); + hv_map_t *ch = hv_struct(&st, 3); + hv_map_t *rl = hv_struct(&st, SKIP_ONE); + hv_map_t *root = hv_struct(&st, PAIR_LEN + SKIP_ONE); + if (ch && rl && root) { + hv_struct_set(ch, "Name", hv_cstr(chart ? chart->name : NULL)); + hv_struct_set(ch, "Version", hv_cstr(chart ? chart->version : NULL)); + hv_struct_set(ch, "AppVersion", hv_cstr(chart ? chart->app_version : NULL)); + hv_struct_set(rl, "Service", hv_cstr("Helm")); + hv_struct_set(root, "Values", hv_json(vals, "")); + hv_struct_set(root, "Chart", (hv_t){.k = HV_MAP, .m = ch}); + hv_struct_set(root, "Release", (hv_t){.k = HV_MAP, .m = rl}); + hv_t dot = {.k = HV_MAP, .m = root}; + hframe_t fr = {.src = src, .dot = dot, .var_base = 0, .out = NULL}; + hst_var_push(&st, "", 0, dot); + he_items(&st, &fr, ts_tree_root_node(tree), NULL); + hst_capture_end(&st); + } + ts_tree_delete(tree); + } + for (int i = 0; i < st.ntrees; i++) { + ts_tree_delete(st.trees[i]); + } + ts_parser_delete(st.parser); + yyjson_mut_doc_free(st.scratch); +} diff --git a/src/pipeline/helm_template.h b/src/pipeline/helm_template.h new file mode 100644 index 000000000..ad24a6f78 --- /dev/null +++ b/src/pipeline/helm_template.h @@ -0,0 +1,83 @@ +/* + * helm_template.h -- Helm values and the rendering of a chart template's + * image references (pass_k8s.c). Internal to the pipeline. + * + * Values are JSON (yyjson mutable values), as Helm holds them after its + * YAML -> JSON step: go-yaml v2 scalars (YAML 1.1 booleans), every number a + * float64. A template renders with Go text/template semantics and the part + * of Sprig image expressions use; anything else -- a function not modelled, + * .Release / .Capabilities / lookup, a missing helper, a parse error on the + * way -- is UNKNOWN, and an image that reads an UNKNOWN is no image. Nothing + * is guessed: a value is what Helm would render, or none. + */ +#ifndef CBM_HELM_TEMPLATE_H +#define CBM_HELM_TEMPLATE_H + +#include "foundation/arena.h" + +#include "tree_sitter/api.h" +#include + +#include +#include + +/* The first document of YAML text src as a value of doc (Helm's scalar + * types); NULL when it does not parse. An empty document is an empty map. */ +yyjson_mut_val *cbm_helm_values_parse(yyjson_mut_doc *doc, const char *src, size_t len); + +/* The YAML node `node` of text `src` (already parsed, e.g. a HelmRelease's + * spec.values) as a value of doc; NULL when it holds an unknown part. */ +yyjson_mut_val *cbm_helm_values_node(yyjson_mut_doc *doc, const char *src, TSNode node); + +/* Helm's coalesce: a deep copy of `base` with `over` merged in -- maps merge + * key by key, any other value replaces, a null deletes. Either may be NULL. */ +yyjson_mut_val *cbm_helm_values_coalesce(yyjson_mut_doc *doc, yyjson_mut_val *base, + yyjson_mut_val *over); + +/* Helm --set NAME=VALUE into map `root`: NAME's `.` separates keys (`\.` + * escapes one), `[n]` indexes a list; VALUE is typed like --set (int, bool, + * null, else string) unless `as_string`. False when NAME is malformed. */ +bool cbm_helm_values_set(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *name, + const char *value, bool as_string); + +/* A values scalar as a template prints it (a number like Go's %v: 1.10 -> + * 1.1) into out; false for a map, a list or null. */ +bool cbm_helm_scalar_text(yyjson_mut_val *v, char *out, size_t sz); + +/* A chart's metadata, as its templates see it under .Chart. */ +typedef struct { + const char *name; + const char *version; + const char *app_version; +} cbm_helm_chart_meta_t; + +/* The named templates (`define` / `block`) of one chart family -- a chart and + * the charts below it share one namespace; for one name the define of the + * file nearest the family root wins (Helm loads deeper files first), then + * the alphabetically first path. */ +typedef struct cbm_helm_defines cbm_helm_defines_t; + +cbm_helm_defines_t *cbm_helm_defines_new(void); +/* Register the defines of template file `rel` (text src, len). The text is + * copied; its parse lives as long as the set. */ +void cbm_helm_defines_add(cbm_helm_defines_t *d, const char *rel, const char *src, size_t len); +void cbm_helm_defines_free(cbm_helm_defines_t *d); + +/* A probe: the `image:` key on a template line. Each time rendering passes + * that line it captures the scalar written after the key. */ +typedef struct { + int line; /* 1-based line of the key in the template */ + int n; /* captures */ + const char **image; /* the rendered scalar, YAML quoting removed; NULL: UNKNOWN */ + const char **keys; /* JSON array of the values paths it read; NULL: none */ +} cbm_helm_probe_t; + +/* Render template `rel` (text src, len) with `values` (its .Values; NULL: + * an empty map), `chart` and the family's defines, filling `probes` (strings + * in arena a). Values paths are dot-joined keys, a key's own `.` escaped + * (`\.`), as --set writes them. */ +void cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, + yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, + const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes); + +#endif /* CBM_HELM_TEMPLATE_H */ diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c index 2fc9e0912..a44a72c9c 100644 --- a/src/pipeline/pass_ci.c +++ b/src/pipeline/pass_ci.c @@ -25,6 +25,7 @@ */ #include "foundation/constants.h" #include "foundation/compat.h" +#include "foundation/str_util.h" /* cbm_json_escape */ #include "pipeline/pipeline_internal.h" #include "cbm.h" #include "helpers.h" /* cbm_env_address */ @@ -783,9 +784,33 @@ static const char *ci_env_lookup(const ci_env_t *env, const char *name, size_t n return NULL; } +/* JSON array `list` (NULL: empty) with identifier `name` appended once. */ +static const char *ci_names_add(CBMArena *a, const char *list, const char *name, size_t nl) { + if (!list) { + return cbm_arena_sprintf(a, "[\"%.*s\"]", (int)nl, name); + } + char quoted[CBM_SZ_256]; + snprintf(quoted, sizeof(quoted), "\"%.*s\"", (int)nl, name); + if (strstr(list, quoted)) { + return list; + } + return cbm_arena_sprintf(a, "%.*s,%s]", (int)(strlen(list) - SKIP_ONE), list, quoted); +} + +static const char *ci_interp_marks(CBMArena *a, const char *s, const ci_env_t *env, + const char **dflt_of, const char **ovr_of); + /* `s` with ${X} / ${X:-d} / ${X-d} / $X expanded from env; NULL when a * variable has no value (and no default). */ static const char *ci_interp(CBMArena *a, const char *s, const ci_env_t *env) { + return ci_interp_marks(a, s, env, NULL, NULL); +} + +/* ci_interp; `dflt_of` (may be NULL) gains, as a JSON array, the variables + * whose inline default was used, `ovr_of` those with an inline default that + * env set instead. */ +static const char *ci_interp_marks(CBMArena *a, const char *s, const ci_env_t *env, + const char **dflt_of, const char **ovr_of) { if (!s || !strchr(s, '$')) { return s; } @@ -839,6 +864,11 @@ static const char *ci_interp(CBMArena *a, const char *s, const ci_env_t *env) { } use = dflt; ul = dl; + if (dflt_of) { + *dflt_of = ci_names_add(a, *dflt_of, name, nl); + } + } else if (dflt && ovr_of) { + *ovr_of = ci_names_add(a, *ovr_of, name, nl); } if (len + ul + SKIP_ONE > cap) { return NULL; @@ -1054,7 +1084,46 @@ static int ci_compose_env(const ci_t *c, TSNode svc, const ci_env_t *dotenv, con return k; } +/* A compose build's `args` (a mapping, or a list of "K=V" items), their + * values interpolated, as a JSON object; NULL: none. An argument without a + * value (the environment's) or with an unknown one is left out. */ +static const char *ci_compose_build_args(const ci_t *c, TSNode build, const ci_env_t *env) { + TSNode args = ts_node_is_null(build) ? build : ci_get(c, build, "args"); + if (ts_node_is_null(args)) { + return NULL; + } + int n = 0; + TSNode *items = ci_items(c, args, &n); + ci_pair_t *pairs = items ? NULL : ci_pairs(c, args, &n); + const char *json = NULL; + for (int i = 0; i < n; i++) { + const char *key = NULL; + const char *raw = NULL; + if (items) { + const char *s = ci_scalar(c, items[i], NULL); + const char *eq = s ? strchr(s, '=') : NULL; + key = eq ? cbm_arena_strndup(c->a, s, (size_t)(eq - s)) : NULL; + raw = eq ? eq + SKIP_ONE : NULL; + } else { + key = pairs[i].key; + raw = ts_node_is_null(pairs[i].value) ? NULL : ci_scalar(c, pairs[i].value, NULL); + } + const char *value = raw ? ci_interp(c->a, raw, env) : NULL; + if (!key || !key[0] || !value) { + continue; + } + char ek[CBM_SZ_256]; + char ev[CBM_SZ_1K]; + cbm_json_escape(ek, (int)sizeof(ek), key); + cbm_json_escape(ev, (int)sizeof(ev), value); + json = cbm_arena_sprintf(c->a, "%s%s\"%s\":\"%s\"", json ? json : "{", json ? "," : "", ek, + ev); + } + return json ? cbm_arena_sprintf(c->a, "%s}", json) : NULL; +} + static void ci_compose(const ci_t *c, TSNode root, const char *dir, const ci_env_t *env) { + static const ci_env_t no_env = {0}; int ns = 0; ci_pair_t *svcs = ci_pairs(c, ci_get(c, root, "services"), &ns); for (int i = 0; i < ns; i++) { @@ -1064,16 +1133,27 @@ static void ci_compose(const ci_t *c, TSNode root, const char *dir, const ci_env } int row = 0; const char *raw = ci_scalar(c, ci_get(c, svc, "image"), &row); - const char *image = raw ? ci_interp(c->a, raw, env) : NULL; - const char *df = ci_compose_dockerfile(c, ci_get(c, svc, "build"), dir, env); + const char *dflt_of = NULL; + const char *ovr_of = NULL; + const char *image = raw ? ci_interp_marks(c->a, raw, env, &dflt_of, &ovr_of) : NULL; + /* With .env replacing a default: what the defaults alone would name. */ + const char *default_image = image && ovr_of ? ci_interp(c->a, raw, &no_env) : NULL; + TSNode build = ci_get(c, svc, "build"); + const char *df = ci_compose_dockerfile(c, build, dir, env); if (c->site_fn && image && df) { - cbm_build_site_t site = { - .via = "compose", .image = image, .dockerfile = df, .line = row + SKIP_ONE}; + cbm_build_site_t site = {.via = "compose", + .image = image, + .dockerfile = df, + .build_args = ci_compose_build_args(c, build, env), + .line = row + SKIP_ONE}; c->site_fn(&site, c->ud); } if (c->service_fn) { cbm_compose_service_t s = {.name = svcs[i].key, .image = image, + .image_default_of = image ? dflt_of : NULL, + .image_override_of = image ? ovr_of : NULL, + .default_image = default_image, .dockerfile = df, .line = (int)ts_node_start_point(svc).row, .end_line = (int)ts_node_end_point(svc).row + SKIP_ONE}; diff --git a/src/pipeline/pass_docker.c b/src/pipeline/pass_docker.c index f5f303866..5d2c2840d 100644 --- a/src/pipeline/pass_docker.c +++ b/src/pipeline/pass_docker.c @@ -108,12 +108,31 @@ typedef struct { CBMHashTable *dirs; /* every directory holding a file (not the root) */ } dk_index_t; +/* Where a value came from: the ARGs whose default (`dflt`) or build argument + * (`ovr`) supplied part of it, as '\x1f'-separated names; NULL: none. */ +typedef struct { + const char *dflt; + const char *ovr; +} dk_prov_t; + typedef struct dk_var { const char *name; const char *value; /* NULL: declared without a value */ + bool arg; /* an ARG: a build argument can set it */ + dk_prov_t prov; struct dk_var *next; } dk_var_t; +/* An image reference of FROM / COPY --from, linked once the file is read. */ +typedef struct { + int64_t img; + const char *ref; + const char *via; + int stage; + int line; + dk_prov_t prov; +} dk_ref_t; + typedef struct { cbm_pipeline_ctx_t *ctx; CBMArena *a; @@ -131,6 +150,14 @@ typedef struct { dk_entry_t entry; /* the current stage's ENTRYPOINT */ dk_entry_t cmd; /* the current stage's CMD */ const dk_index_t *ix; + dk_ref_t *refs; /* FROM / COPY --from references, in file order */ + int nrefs; + int refs_cap; + /* A build's view (dk_build_args): its build arguments set the ARGs they + * name, and the references they change link from the Image it builds. */ + const dk_var_t *build_args; + int64_t built; /* 0: the Dockerfile's own view */ + const char *build_props; /* the build's tool / file / build_line members */ } dk_t; static bool dk_type(TSNode n, const char *t) { @@ -172,14 +199,72 @@ static const dk_var_t *dk_find(const dk_var_t *v, const char *name, size_t nl) { return NULL; } -static void dk_set(dk_t *d, dk_var_t **list, const char *name, const char *value) { +static void dk_set(dk_t *d, dk_var_t **list, const char *name, const char *value, bool arg, + dk_prov_t prov) { dk_var_t *v = cbm_arena_alloc(d->a, sizeof(*v)); if (v) { - *v = (dk_var_t){.name = name, .value = value, .next = *list}; + *v = (dk_var_t){.name = name, .value = value, .arg = arg, .prov = prov, .next = *list}; *list = v; } } +enum { DK_NAME_SEP = '\x1f' }; + +static bool dk_names_has(const char *list, const char *name, size_t nl) { + for (const char *p = list; p && *p;) { + size_t l = strcspn(p, "\x1f"); + if (l == nl && strncmp(p, name, nl) == 0) { + return true; + } + p += l; + p += *p ? SKIP_ONE : 0; + } + return false; +} + +/* `list` with `name` appended unless it holds it already. */ +static const char *dk_names_add(dk_t *d, const char *list, const char *name, size_t nl) { + if (dk_names_has(list, name, nl)) { + return list; + } + return list && list[0] ? cbm_arena_sprintf(d->a, "%s%c%.*s", list, DK_NAME_SEP, (int)nl, name) + : cbm_arena_strndup(d->a, name, nl); +} + +static const char *dk_names_union(dk_t *d, const char *a, const char *b) { + for (const char *p = b; p && *p;) { + size_t l = strcspn(p, "\x1f"); + a = dk_names_add(d, a, p, l); + p += l; + p += *p ? SKIP_ONE : 0; + } + return a; +} + +static void dk_prov_merge(dk_t *d, dk_prov_t *into, dk_prov_t from) { + into->dflt = dk_names_union(d, into->dflt, from.dflt); + into->ovr = dk_names_union(d, into->ovr, from.ovr); +} + +/* `,"key":["A","B"]` for a non-empty name list, else "". */ +static const char *dk_names_member(dk_t *d, const char *key, const char *list) { + if (!list || !list[0]) { + return ""; + } + const char *out = cbm_arena_sprintf(d->a, ",\"%s\":[", key); + for (const char *p = list; out && *p;) { + size_t l = strcspn(p, "\x1f"); + char esc[CBM_SZ_256]; + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "%.*s", (int)l, p); + cbm_json_escape(esc, (int)sizeof(esc), name); + out = cbm_arena_sprintf(d->a, "%s%s\"%s\"", out, p == list ? "" : ",", esc); + p += l; + p += *p ? SKIP_ONE : 0; + } + return out ? cbm_arena_sprintf(d->a, "%s]", out) : ""; +} + /* ASCII case-insensitive equality (stage names, `scratch`). */ static bool dk_ieq(const char *a, const char *b) { for (; *a && *b; a++, b++) { @@ -195,8 +280,10 @@ static bool dk_name_char(char c) { } /* `text` with $X / ${X} / ${X:-d} / ${X-d} expanded from `vars`; NULL when a - * reference has no value. */ -static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text) { + * reference has no value. `prov` (may be NULL) gains the ARGs whose default + * or build argument supplied a value -- an inline `${X:-d}` default counts + * only for a declared ARG X, the one thing a build argument can set. */ +static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text, dk_prov_t *prov) { size_t cap = strlen(text) + SKIP_ONE; size_t len = 0; char *out = cbm_arena_alloc(d->a, cap); @@ -249,6 +336,11 @@ static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text) { return NULL; } const char *from = val ? val : dflt; + if (prov && v && val) { + dk_prov_merge(d, prov, v->prov); + } else if (prov && v && v->arg) { + prov->dflt = dk_names_add(d, prov->dflt, name, nl); + } if (len + vl + SKIP_ONE > cap) { size_t ncap = (len + vl + SKIP_ONE) * PAIR_LEN; char *grown = cbm_arena_alloc(d->a, ncap); @@ -370,6 +462,139 @@ static int dk_stage_of(const dk_t *d, const char *ref) { return found; } +/* A path component: [a-z0-9]+ joined by `.`, `_`, `__` or `-`+. */ +static bool dk_ref_component(const char *s, size_t n) { + if (n == 0 || !isalnum((unsigned char)s[0]) || !isalnum((unsigned char)s[n - SKIP_ONE])) { + return false; + } + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (isupper(c)) { + return false; + } + if (isalnum(c)) { + continue; + } + size_t run = SKIP_ONE; + while (i + run < n && s[i + run] == c) { + run++; + } + if (!((c == '.' && run == SKIP_ONE) || (c == '_' && run <= PAIR_LEN) || c == '-')) { + return false; + } + i += run - SKIP_ONE; + } + return true; +} + +/* A registry host: DNS labels (`.`-joined), or an IPv6 literal, with an + * optional :port. */ +static bool dk_ref_domain(const char *s, size_t n) { + const char *colon = memchr(s, ':', n); + if (n && s[0] == '[') { + const char *close = memchr(s, ']', n); + colon = close && (size_t)(close - s) + SKIP_ONE < n ? close + SKIP_ONE : NULL; + if (!close) { + return false; + } + } + size_t hl = colon ? (size_t)(colon - s) : n; + if (hl == 0) { + return false; + } + if (s[0] != '[') { + for (size_t i = 0; i < hl; i++) { + unsigned char c = (unsigned char)s[i]; + if (!isalnum(c) && c != '.' && c != '-') { + return false; + } + } + } + for (size_t i = hl + SKIP_ONE; colon && i < n; i++) { + if (!isdigit((unsigned char)s[i])) { + return false; + } + } + return !colon || hl + SKIP_ONE < n; +} + +bool cbm_image_ref_valid(const char *ref) { + if (!ref || !ref[0]) { + return false; + } + size_t n = strlen(ref); + const char *at = strchr(ref, '@'); + if (at) { + const char *colon = strchr(at, ':'); + if (!colon || colon == at + SKIP_ONE || strlen(colon + SKIP_ONE) < CBM_SZ_32) { + return false; + } + for (const char *p = at + SKIP_ONE; p < colon; p++) { + if (!isalnum((unsigned char)*p) && !strchr("+.-_", *p)) { + return false; + } + } + for (const char *p = colon + SKIP_ONE; *p; p++) { + if (!isxdigit((unsigned char)*p)) { + return false; + } + } + n = (size_t)(at - ref); + } + /* the tag: after the last ':' that follows the last '/' */ + size_t name_end = n; + for (size_t i = n; i > 0; i--) { + if (ref[i - SKIP_ONE] == '/') { + break; + } + if (ref[i - SKIP_ONE] == ':') { + name_end = i - SKIP_ONE; + break; + } + } + if (name_end < n) { + size_t tl = n - name_end - SKIP_ONE; + const char *tag = ref + name_end + SKIP_ONE; + if (tl == 0 || tl > CBM_SZ_128 || !(isalnum((unsigned char)tag[0]) || tag[0] == '_')) { + return false; + } + for (size_t i = 0; i < tl; i++) { + if (!isalnum((unsigned char)tag[i]) && !strchr("_.-", tag[i])) { + return false; + } + } + } + const char *p = ref; + const char *end = ref + name_end; + const char *slash = memchr(p, '/', (size_t)(end - p)); + if (slash) { + size_t fl = (size_t)(slash - p); + bool host = memchr(p, '.', fl) || memchr(p, ':', fl) || + (fl == strlen("localhost") && memcmp(p, "localhost", fl) == 0); + if (host) { + if (!dk_ref_domain(p, fl)) { + return false; + } + p = slash + SKIP_ONE; + } + } + if (p >= end) { + return false; + } + while (p < end) { + const char *s = memchr(p, '/', (size_t)(end - p)); + size_t cl = s ? (size_t)(s - p) : (size_t)(end - p); + if (!dk_ref_component(p, cl)) { + return false; + } + p += cl + (s ? SKIP_ONE : 0); + if (s && p >= end) { + return false; + } + } + return true; +} + int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref) { char repo[CBM_SZ_512]; char reg[CBM_SZ_512]; @@ -438,19 +663,101 @@ int64_t cbm_pipeline_env_setter(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t * return id; } -/* DEPENDS_ON from the Dockerfile to the Image of resolved reference `ref`. */ -static void dk_link(dk_t *d, const char *ref, const char *via, int line) { +/* Record the Image of resolved reference `ref`, linked by dk_refs_link. */ +static void dk_link(dk_t *d, const char *ref, const char *via, int line, dk_prov_t prov) { int64_t img = cbm_pipeline_image_node(d->ctx, ref); if (img <= 0) { return; } - char esc_ref[CBM_SZ_1K]; - cbm_json_escape(esc_ref, (int)sizeof(esc_ref), ref); - char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), "{\"via\":\"%s\",\"image\":\"%s\",\"stage\":%d,\"line\":%d}", - via, esc_ref, d->stage, line); - cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, img, "DEPENDS_ON", props); - d->edges++; + if (d->nrefs == d->refs_cap) { + int cap = d->refs_cap ? d->refs_cap * PAIR_LEN : DK_MAX_STAGES; + dk_ref_t *grown = cbm_arena_alloc(d->a, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + if (d->nrefs) { + memcpy(grown, d->refs, (size_t)d->nrefs * sizeof(*grown)); + } + d->refs = grown; + d->refs_cap = cap; + } + d->refs[d->nrefs++] = (dk_ref_t){img, ref, via, d->stage, line, prov}; +} + +/* The names a default supplies to the references of Image `img`: their union, + * or NULL when any of them names it without a default (a literal reference + * wins -- the edges to one Image are one edge, so every one of them carries + * the same mark). */ +static const char *dk_refs_defaults(dk_t *d, int64_t img) { + const char *names = NULL; + for (int i = 0; i < d->nrefs; i++) { + if (d->refs[i].img != img) { + continue; + } + if (!d->refs[i].prov.dflt) { + return NULL; + } + names = dk_names_union(d, names, d->refs[i].prov.dflt); + } + return names; +} + +/* The Dockerfile's own view: DEPENDS_ON from its Module to each Image FROM / + * COPY --from names {via, image, stage, line[, default_of: the ARGs whose + * defaults named it]}. */ +static void dk_refs_link(dk_t *d) { + for (int i = 0; i < d->nrefs; i++) { + const dk_ref_t *r = &d->refs[i]; + char esc_ref[CBM_SZ_1K]; + cbm_json_escape(esc_ref, (int)sizeof(esc_ref), r->ref); + const char *dflt = dk_names_member(d, "default_of", dk_refs_defaults(d, r->img)); + const char *props = + cbm_arena_sprintf(d->a, "{\"via\":\"%s\",\"image\":\"%s\",\"stage\":%d,\"line\":%d%s}", + r->via, esc_ref, r->stage, r->line, dflt); + if (props) { + cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, r->img, "DEPENDS_ON", props); + d->edges++; + } + } +} + +/* A build's view: each Image a reference the build's arguments changed names + * -- the first reference to it, its build arguments unioned -- + * DEPENDS_ON from the Image the build makes {via, image, override_of[, + * default_of], dockerfile, stage, line, tool, file, build_line}. A reference + * no build argument reached is the Dockerfile's own edge already. */ +static void dk_refs_link_build(dk_t *d) { + for (int i = 0; i < d->nrefs; i++) { + const dk_ref_t *r = &d->refs[i]; + bool first = r->prov.ovr != NULL && r->img != d->built; + for (int j = 0; first && j < i; j++) { + first = !(d->refs[j].img == r->img && d->refs[j].prov.ovr); + } + if (!first) { + continue; + } + dk_prov_t all = {0}; + for (int j = i; j < d->nrefs; j++) { + if (d->refs[j].img == r->img && d->refs[j].prov.ovr) { + dk_prov_merge(d, &all, d->refs[j].prov); + } + } + char esc_ref[CBM_SZ_1K]; + char esc_file[CBM_SZ_1K]; + cbm_json_escape(esc_ref, (int)sizeof(esc_ref), r->ref); + cbm_json_escape(esc_file, (int)sizeof(esc_file), d->file); + const char *props = cbm_arena_sprintf( + d->a, + "{\"via\":\"%s\",\"image\":\"%s\"%s%s,\"dockerfile\":\"%s\",\"stage\":%d," + "\"line\":%d,%s}", + r->via, esc_ref, dk_names_member(d, "override_of", all.ovr), + dk_names_member(d, "default_of", all.dflt), esc_file, r->stage, r->line, + d->build_props); + if (props) { + cbm_gbuf_insert_edge(d->ctx->gbuf, d->built, r->img, "DEPENDS_ON", props); + d->edges++; + } + } } static int dk_line(TSNode n) { @@ -467,19 +774,33 @@ static void dk_arg(dk_t *d, TSNode ins) { if (!name) { return; } - if (d->stage < 0) { - dk_set(d, &d->global, name, ts_node_is_null(df) ? NULL : dk_value(d, df)); + size_t nl = strlen(name); + dk_var_t **list = d->stage < 0 ? &d->global : &d->local; + /* A build argument sets the ARG it names, in every stage declaring it, + * whatever its default. */ + const dk_var_t *set = dk_find(d->build_args, name, nl); + if (set) { + dk_set(d, list, name, set->value, true, + (dk_prov_t){.ovr = dk_names_add(d, NULL, name, nl)}); return; } const char *value = NULL; + dk_prov_t prov = {0}; if (!ts_node_is_null(df)) { + /* A default may name ARGs declared before it (ARG BASE=golang:${GO}). */ const char *raw = dk_value(d, df); - value = raw ? dk_expand(d, d->local, raw) : NULL; - } else { - const dk_var_t *g = dk_find(d->global, name, strlen(name)); - value = g ? g->value : NULL; + value = raw ? dk_expand(d, *list, raw, &prov) : NULL; + if (value) { + prov.dflt = dk_names_add(d, prov.dflt, name, nl); + } + } else if (d->stage >= 0) { + const dk_var_t *g = dk_find(d->global, name, nl); + if (g) { + value = g->value; + prov = g->prov; + } } - dk_set(d, &d->local, name, value); + dk_set(d, list, name, value, true, prov); } static void dk_env(dk_t *d, TSNode ins) { @@ -498,7 +819,9 @@ static void dk_env(dk_t *d, TSNode ins) { } const char *raw = ts_node_is_null(vl) ? NULL : dk_value(d, vl); const char *name = dk_text(d, nm); - dk_set(d, &d->local, name, raw ? dk_expand(d, d->local, raw) : NULL); + dk_prov_t prov = {0}; + const char *value = raw ? dk_expand(d, d->local, raw, &prov) : NULL; + dk_set(d, &d->local, name, value, false, prov); dk_envkey_t *k = d->stage < DK_MAX_STAGES ? cbm_arena_alloc(d->a, sizeof(*k)) : NULL; if (k && name) { *k = (dk_envkey_t){name, dk_line(pair), d->st[d->stage].envs}; @@ -515,7 +838,9 @@ static void dk_from(dk_t *d, TSNode ins) { spec = ts_node_named_child(ins, i); } } - const char *ref = ts_node_is_null(spec) ? NULL : dk_expand(d, d->global, dk_text(d, spec)); + dk_prov_t prov = {0}; + const char *ref = + ts_node_is_null(spec) ? NULL : dk_expand(d, d->global, dk_text(d, spec), &prov); d->stage++; d->local = NULL; d->workdir = "/"; @@ -523,7 +848,7 @@ static void dk_from(dk_t *d, TSNode ins) { d->cmd.set = false; bool stage = ref && dk_is_stage(d, ref); if (ref && !stage) { - dk_link(d, ref, "from", dk_line(ins)); + dk_link(d, ref, "from", dk_line(ins), prov); } if (d->stage < DK_MAX_STAGES) { d->st[d->stage].parent = stage ? dk_stage_of(d, ref) : -1; @@ -605,7 +930,7 @@ static int dk_copy_json(dk_t *d, const char *p, const char **out, int max) { if (yyjson_is_arr(arr)) { for (yyjson_arr_iter_init(arr, &it); n < max && (v = yyjson_arr_iter_next(&it));) { const char *s = yyjson_get_str(v); - out[n++] = s ? dk_expand(d, d->local, cbm_arena_strdup(d->a, s)) : NULL; + out[n++] = s ? dk_expand(d, d->local, cbm_arena_strdup(d->a, s), NULL) : NULL; } } if (doc) { @@ -639,11 +964,12 @@ static void dk_copy(dk_t *d, TSNode ins) { if (l > sizeof(FROM) - SKIP_ONE && strncmp(p, FROM, sizeof(FROM) - SKIP_ONE) == 0) { const char *v = cbm_arena_strndup(d->a, p + sizeof(FROM) - SKIP_ONE, l - (sizeof(FROM) - SKIP_ONE)); - const char *ref = v ? dk_expand(d, d->local, v) : NULL; + dk_prov_t prov = {0}; + const char *ref = v ? dk_expand(d, d->local, v, &prov) : NULL; c->has_from = true; c->from = ref; if (ref && ref[0] && !dk_is_stage(d, ref)) { - dk_link(d, ref, "copy-from", dk_line(ins)); + dk_link(d, ref, "copy-from", dk_line(ins), prov); } } p += l; @@ -652,7 +978,7 @@ static void dk_copy(dk_t *d, TSNode ins) { while (!json && *p && nw < DK_MAX_WORDS) { size_t l = strcspn(p, " \t"); const char *w = cbm_arena_strndup(d->a, p, l); - words[nw++] = w && strncmp(w, "<<", PAIR_LEN) != 0 ? dk_expand(d, d->local, w) : NULL; + words[nw++] = w && strncmp(w, "<<", PAIR_LEN) != 0 ? dk_expand(d, d->local, w, NULL) : NULL; p += l; p += strspn(p, " \t"); } @@ -1023,8 +1349,8 @@ static void dk_go_builds(dk_t *d, const char **w, int n) { if (!b) { return; } - *b = (dk_build_t){.out = out ? dk_expand(d, d->local, out) : NULL, - .pkg = dk_expand(d, d->local, pkg ? pkg : "."), + *b = (dk_build_t){.out = out ? dk_expand(d, d->local, out, NULL) : NULL, + .pkg = dk_expand(d, d->local, pkg ? pkg : ".", NULL), .workdir = d->workdir, .next = d->st[d->stage].builds}; d->st[d->stage].builds = b; @@ -1066,7 +1392,7 @@ static void dk_workdir(dk_t *d, TSNode ins) { continue; } const char *raw = dk_text(d, pn); - const char *v = raw ? dk_expand(d, d->local, raw) : NULL; + const char *v = raw ? dk_expand(d, d->local, raw, NULL) : NULL; const char *w = v ? v : raw; if (w) { d->workdir = dk_absolute(d, w, d->workdir); @@ -1431,11 +1757,31 @@ static void dk_file(dk_t *d, TSNode root) { dk_entry(d, ins, &d->cmd); } } + dk_refs_link(d); dk_spawn(d); dk_envs(d); dk_image_set(d); } +/* A build's view of the Dockerfile: only what decides its image references + * (ARG, ENV, FROM, COPY --from) is read, under the build's arguments. */ +static void dk_file_build(dk_t *d, TSNode root) { + for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { + TSNode ins = ts_node_named_child(root, i); + if (dk_type(ins, "arg_instruction")) { + dk_arg(d, ins); + } else if (dk_type(ins, "env_instruction")) { + dk_env(d, ins); + } else if (dk_type(ins, "from_instruction")) { + dk_from(d, ins); + } else if ((dk_type(ins, "copy_instruction") || dk_type(ins, "add_instruction")) && + d->stage >= 0) { + dk_copy(d, ins); + } + } + dk_refs_link_build(d); +} + static char *dk_read(const char *path, size_t *len) { FILE *f = cbm_fopen(path, "rb"); if (!f) { @@ -1545,12 +1891,15 @@ static void dk_build_site(const cbm_build_site_t *s, void *ud) { char esc_img[CBM_SZ_1K]; cbm_json_escape(esc_file, (int)sizeof(esc_file), b->file); cbm_json_escape(esc_img, (int)sizeof(esc_img), s->image); - char props[CBM_SZ_4K]; - snprintf(props, sizeof(props), - "{\"via\":\"build\",\"tool\":\"%s\",\"file\":\"%s\",\"image\":\"%s\",\"line\":%d}", - s->via, esc_file, esc_img, s->line); - cbm_gbuf_insert_edge(b->ctx->gbuf, img, mod->id, "DEPENDS_ON", props); - b->edges++; + const char *props = cbm_arena_sprintf( + b->a, + "{\"via\":\"build\",\"tool\":\"%s\",\"file\":\"%s\",\"image\":\"%s\",\"line\":%d%s%s}", + s->via, esc_file, esc_img, s->line, s->build_args ? ",\"build_args\":" : "", + s->build_args ? s->build_args : ""); + if (props) { + cbm_gbuf_insert_edge(b->ctx->gbuf, img, mod->id, "DEPENDS_ON", props); + b->edges++; + } } typedef struct { @@ -1599,6 +1948,41 @@ static void dk_compose_connects(dk_compose_t *c, const cbm_compose_service_t *s, * DEFINES; it DEPENDS_ON the services it depends on / links, USAGE * {ref:image} its image, DEPENDS_ON {via:build} the Dockerfile it builds and * CONNECTS_TO the services its environment addresses. */ +/* The image a service runs: USAGE {ref: image, image[, default_of]} -- the + * variables whose inline default named it. When the project's .env replaced + * a default, the edge carries {override_of, override_source: ".env"}, and + * the image the defaults alone name is kept: on the same Image as + * `default_image`, else its own USAGE {ref: image_default, default_of}, a + * reference the service does not run. */ +static void dk_compose_image(dk_compose_t *c, const cbm_compose_service_t *s, int64_t self) { + int64_t img = cbm_pipeline_image_node(c->ctx, s->image); + int64_t dimg = s->default_image ? cbm_pipeline_image_node(c->ctx, s->default_image) : 0; + char esc[CBM_SZ_1K]; + char desc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), s->image); + cbm_json_escape(desc, (int)sizeof(desc), s->default_image ? s->default_image : ""); + const char *props = + cbm_arena_sprintf(c->a, "{\"ref\":\"image\",\"image\":\"%s\"%s%s%s%s%s%s%s}", esc, + s->image_default_of ? ",\"default_of\":" : "", + s->image_default_of ? s->image_default_of : "", + s->image_override_of ? ",\"override_of\":" : "", + s->image_override_of ? s->image_override_of : "", + s->image_override_of ? ",\"override_source\":\".env\"" : "", + dimg > 0 && dimg == img ? ",\"default_image\":\"" : "", + dimg > 0 && dimg == img ? cbm_arena_sprintf(c->a, "%s\"", desc) : ""); + if (props) { + dk_compose_edge(c, self, img, "USAGE", props); + } + if (dimg > 0 && dimg != img && s->image_override_of) { + const char *dprops = cbm_arena_sprintf( + c->a, "{\"ref\":\"image_default\",\"image\":\"%s\",\"default_of\":%s}", desc, + s->image_override_of); + if (dprops) { + dk_compose_edge(c, self, dimg, "USAGE", dprops); + } + } +} + static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { dk_compose_t *c = ud; const char *qn = dk_compose_qn(c, s->name); @@ -1635,11 +2019,7 @@ static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { } dk_compose_connects(c, s, self->id); if (s->image) { - char esc[CBM_SZ_1K]; - cbm_json_escape(esc, (int)sizeof(esc), s->image); - char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), "{\"ref\":\"image\",\"image\":\"%s\"}", esc); - dk_compose_edge(c, self->id, cbm_pipeline_image_node(c->ctx, s->image), "USAGE", props); + dk_compose_image(c, s, self->id); } if (s->dockerfile) { const char *mqn = cbm_fqn_module(c->a, c->ctx->project_name, s->dockerfile); @@ -1706,6 +2086,152 @@ static int dk_link_builds(cbm_pipeline_ctx_t *ctx, CBMArena *a) { return b.edges; } +static bool dk_prop(const char *props, const char *key, char *out, size_t out_sz); +static void *dk_grow(CBMArena *a, void *v, int n, int *cap, size_t size); + +/* A build that passes build arguments (compose build.args, docker build + * --build-arg; on its build edge as `build_args`). */ +typedef struct { + int64_t img; + int64_t module; + const char *dockerfile; + const dk_var_t *args; + const char *props; /* "tool":..,"file":..,"build_line":N */ +} dk_argbuild_t; + +/* The build arguments of build edge properties `props` (NULL: none). */ +static const dk_var_t *dk_build_args_of(CBMArena *a, const char *props) { + if (!props || !strstr(props, "\"build_args\"")) { + return NULL; + } + yyjson_doc *doc = yyjson_read(props, strlen(props), 0); + yyjson_val *args = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "build_args") : NULL; + dk_var_t *list = NULL; + yyjson_obj_iter it; + yyjson_val *k; + if (yyjson_is_obj(args)) { + yyjson_obj_iter_init(args, &it); + while ((k = yyjson_obj_iter_next(&it))) { + const char *v = yyjson_get_str(yyjson_obj_iter_get_val(k)); + dk_var_t *var = v ? cbm_arena_alloc(a, sizeof(*var)) : NULL; + if (var) { + *var = (dk_var_t){.name = cbm_arena_strdup(a, yyjson_get_str(k)), + .value = cbm_arena_strdup(a, v), + .arg = true, + .next = list}; + list = var; + } + } + } + if (doc) { + yyjson_doc_free(doc); + } + return list; +} + +/* Every build edge (Image -DEPENDS_ON {via: build}-> Dockerfile) that passes + * build arguments, into out (arena); returns how many. */ +static int dk_argbuilds(cbm_pipeline_ctx_t *ctx, CBMArena *a, dk_argbuild_t **out) { + const cbm_gbuf_node_t **imgs = NULL; + int ni = 0; + *out = NULL; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Image", &imgs, &ni) != 0) { + return 0; + } + int n = 0; + int cap = 0; + for (int i = 0; i < ni; i++) { + const cbm_gbuf_edge_t **es = NULL; + int ne = 0; + if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, imgs[i]->id, "DEPENDS_ON", &es, &ne) != + 0) { + continue; + } + for (int j = 0; j < ne; j++) { + const char *p = es[j]->properties_json; + char via[CBM_SZ_32]; + const dk_var_t *args = NULL; + if (!dk_prop(p, "via", via, sizeof(via)) || strcmp(via, "build") != 0 || + !(args = dk_build_args_of(a, p))) { + continue; + } + const cbm_gbuf_node_t *mod = cbm_gbuf_find_by_id(ctx->gbuf, es[j]->target_id); + char tool[CBM_SZ_64]; + char file[CBM_SZ_1K]; + char esc[CBM_SZ_2K]; + if (!mod || !mod->file_path || !dk_prop(p, "tool", tool, sizeof(tool)) || + !dk_prop(p, "file", file, sizeof(file))) { + continue; + } + const char *line = strstr(p, "\"line\":"); + if (n == cap) { + *out = dk_grow(a, *out, n, &cap, sizeof(**out)); + if (!*out) { + return 0; + } + } + cbm_json_escape(esc, (int)sizeof(esc), file); + (*out)[n++] = (dk_argbuild_t){ + .img = imgs[i]->id, + .module = mod->id, + .dockerfile = cbm_arena_strdup(a, mod->file_path), + .args = args, + .props = cbm_arena_sprintf(a, "\"tool\":\"%s\",\"file\":\"%s\",\"build_line\":%d", + tool, esc, line ? atoi(line + strlen("\"line\":")) : 0)}; + } + } + return n; +} + +/* Each build passing build arguments reads its Dockerfile in its own view + * (dk_file_build): what those arguments change, it builds FROM. Image-sourced + * DEPENDS_ON edges, recomputed whole every run (CBM_BUILD_EDGE_SOURCE). */ +static int dk_link_build_args(cbm_pipeline_ctx_t *ctx, CBMArena *a) { + dk_argbuild_t *bs = NULL; + int n = ctx->repo_path ? dk_argbuilds(ctx, a, &bs) : 0; + if (n == 0) { + return 0; + } + TSParser *parser = ts_parser_new(); + if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_DOCKERFILE))) { + if (parser) { + ts_parser_delete(parser); + } + return 0; + } + int edges = 0; + for (int i = 0; i < n; i++) { + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, bs[i].dockerfile); + size_t len = 0; + char *src = dk_read(path, &len); + if (!src) { + continue; + } + dk_uncomment(src, len); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + if (tree) { + dk_t d = {.ctx = ctx, + .a = a, + .src = src, + .file = bs[i].dockerfile, + .module = bs[i].module, + .stage = -1, + .dir = "", + .workdir = "/", + .build_args = bs[i].args, + .built = bs[i].img, + .build_props = bs[i].props}; + dk_file_build(&d, ts_tree_root_node(tree)); + edges += d.edges; + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + ts_parser_delete(parser); + return edges; +} + static bool dk_node_before(const cbm_gbuf_node_t *a, const cbm_gbuf_node_t *b) { int c = strcmp(a->file_path, b->file_path); return c < 0 || (c == 0 && a->start_line < b->start_line); @@ -1891,17 +2417,19 @@ static void dk_hub_push(CBMArena *a, dk_hub_edges_t *l, const cbm_gbuf_edge_t *e * USAGE {ref:image}, or DEPENDS_ON {via:from|copy-from}? */ static bool dk_hub_ref(const char *type, const char *p) { char v[CBM_SZ_32]; - if (strcmp(type, "USAGE") == 0) { + if (strcmp(type, "USAGE") == 0 || strcmp(type, "DEPLOYS") == 0) { return dk_prop(p, "ref", v, sizeof(v)) && strcmp(v, "image") == 0; } return dk_prop(p, "via", v, sizeof(v)) && (strcmp(v, "from") == 0 || strcmp(v, "copy-from") == 0); } -/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}) and - * references in, each with the tag of its `image` reference. */ +/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}), the + * base images a build's arguments chose out (DEPENDS_ON {via: from | + * copy-from, build_line}: they follow their build) and references in, each + * with the tag of its `image` reference. */ static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, - dk_hub_edges_t *builds, dk_hub_edges_t *refs) { + dk_hub_edges_t *builds, dk_hub_edges_t *refs, dk_hub_edges_t *follows) { const cbm_gbuf_edge_t **es = NULL; int n = 0; char ref[CBM_SZ_1K]; @@ -1909,8 +2437,13 @@ static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, img, "DEPENDS_ON", &es, &n) == 0) { for (int i = 0; i < n; i++) { char via[CBM_SZ_32]; - if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via)) || - strcmp(via, "build") != 0) { + if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via))) { + continue; + } + if (strcmp(via, "build") != 0) { + if (strstr(es[i]->properties_json, "\"build_line\":")) { + dk_hub_push(a, follows, es[i], ""); + } continue; } tag[0] = '\0'; @@ -1921,7 +2454,7 @@ static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, } atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed); } - static const char *const REF_TYPES[] = {"USAGE", "DEPENDS_ON"}; + static const char *const REF_TYPES[] = {"USAGE", "DEPENDS_ON", "DEPLOYS"}; for (size_t t = 0; t < sizeof(REF_TYPES) / sizeof(REF_TYPES[0]); t++) { if (cbm_gbuf_find_edges_by_target_type(ctx->gbuf, img, REF_TYPES[t], &es, &n) != 0) { continue; @@ -2122,6 +2655,27 @@ static int dk_hub_move(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_hub_edge_t return SKIP_ONE; } +static int dk_prop_int(const char *props, const char *key) { + char pat[CBM_SZ_64]; + snprintf(pat, sizeof(pat), "\"%s\":", key); + const char *p = props ? strstr(props, pat) : NULL; + return p ? atoi(p + strlen(pat)) : -SKIP_ONE; +} + +/* Does base-image edge `f` come from build edge `b`: one source, and its + * tool / file / build_line those of the build? */ +static bool dk_hub_follows(const dk_hub_edge_t *f, const dk_hub_edge_t *b) { + char ft[CBM_SZ_64]; + char bt[CBM_SZ_64]; + char ff[CBM_SZ_1K]; + char bf[CBM_SZ_1K]; + return f->src == b->src && dk_prop(f->props, "tool", ft, sizeof(ft)) && + dk_prop(b->props, "tool", bt, sizeof(bt)) && strcmp(ft, bt) == 0 && + dk_prop(f->props, "file", ff, sizeof(ff)) && dk_prop(b->props, "file", bf, sizeof(bf)) && + strcmp(ff, bf) == 0 && + dk_prop_int(f->props, "build_line") == dk_prop_int(b->props, "line"); +} + /* One repository's Images (imgs[0..n)): split it by component when it is * built from several Dockerfiles, else keep (or bring back) everything on * the repository Image. Returns the edges moved. */ @@ -2129,10 +2683,11 @@ static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *img dk_drop_t *drop) { dk_hub_edges_t builds = {0}; dk_hub_edges_t refs = {0}; + dk_hub_edges_t follows = {0}; bool has_sub = false; for (int i = 0; i < n; i++) { has_sub = has_sub || strcmp(imgs[i].name, imgs[i].repo) != 0; - dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs); + dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs, &follows); } const char **comps = NULL; size_t pre = 0; @@ -2158,6 +2713,11 @@ static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *img int k = builds.v[i].comp; int64_t src = sub && k >= 0 && sub[k] > 0 ? sub[k] : repo_img; moved += dk_hub_move(ctx, a, &builds.v[i], src, builds.v[i].tgt, drop); + for (int f = 0; f < follows.n; f++) { + if (dk_hub_follows(&follows.v[f], &builds.v[i])) { + moved += dk_hub_move(ctx, a, &follows.v[f], src, follows.v[f].tgt, drop); + } + } } for (int i = 0; i < refs.n; i++) { int k = sub ? dk_hub_match(refs.v[i].tag, comps, nc) : -1; @@ -2274,6 +2834,7 @@ void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fi cbm_ht_free(ix.dirs); } edges += dk_link_builds(ctx, &arena); + edges += dk_link_build_args(ctx, &arena); edges += dk_image_hubs(ctx, &arena); cbm_arena_destroy(&arena); if (edges > 0) { diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index a9e92acc7..60433514c 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -20,10 +20,12 @@ #include "foundation/compat_fs.h" #include "foundation/limits.h" #include "cbm.h" -#include "helpers.h" /* cbm_kind_in_set_free_cache */ -#include "foundation/slab_alloc.h" /* cbm_slab_destroy_thread */ -#include "foundation/platform.h" /* cbm_default_worker_count */ -#include "foundation/str_util.h" /* cbm_json_escape */ +#include "helpers.h" /* cbm_kind_in_set_free_cache */ +#include "foundation/slab_alloc.h" /* cbm_slab_destroy_thread */ +#include "foundation/platform.h" /* cbm_default_worker_count */ +#include "foundation/str_util.h" /* cbm_json_escape */ +#include "foundation/json_writer.h" /* cbm_json_writer_allocator */ +#include "pipeline/helm_template.h" #include "pipeline/worker_pool.h" #include @@ -1103,9 +1105,7 @@ static void k8s_bundles_free(k8s_bundles_t *b) { } /* A Helm chart's directory bundles the plain manifests below it. */ -static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { - char dir[CBM_SZ_1K]; - k8s_dirname(chart_yaml, dir, sizeof(dir)); +static void k8s_chart_dir_register(k8s_bundles_t *b, const char *dir) { if (b->charts && !cbm_ht_get(b->charts, dir)) { char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); if (owned) { @@ -1114,6 +1114,12 @@ static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { } } +static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { + char dir[CBM_SZ_1K]; + k8s_dirname(chart_yaml, dir, sizeof(dir)); + k8s_chart_dir_register(b, dir); +} + /* Build the kustomize bundles (call once every manifest and kustomization is * known and the IMPORTS edges exist). */ static void k8s_bundles_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b) { @@ -1285,6 +1291,9 @@ typedef struct { int64_t node; char *image; char *container; /* NULL: unnamed */ + int line; /* the image's line */ + int start_line; /* the container's lines */ + int end_line; } k8s_img_t; /* A patch a kustomization states (extract_k8s.c PATCH fact): what it names @@ -1482,8 +1491,12 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 } else if (x->fact == CBM_K8S_FACT_IMAGE) { if (x->name && k8s_grow((void **)&f->imgs, &f->img_cap, f->img_n, sizeof(*f->imgs))) { f->imgs[f->img_n++] = (k8s_img_t){ - node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), - x->container ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) : NULL}; + node, + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + x->container ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) : NULL, + x->line, + x->start_line, + x->end_line}; } } else if (x->fact == CBM_K8S_FACT_ENV) { if (x->name && k8s_grow((void **)&f->envs, &f->env_cap, f->env_n, sizeof(*f->envs))) { @@ -2772,6 +2785,27 @@ static void k8s_link_local_deps(cbm_pipeline_ctx_t *ctx, k8s_local_deps_t *deps) * `file://` dependency (once all charts are read), otherwise to the external * Chart node of the dependency's repository and name, the repository also on * the edge. */ +/* The Chart node `name` defined by chart file `rel_path` (a Chart.yaml, a + * werf.yaml), its File node DEFINES it; 0 on failure. */ +static int64_t k8s_chart_upsert(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *name, + const char *props) { + char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); + if (!chart_qn) { + return 0; + } + int64_t chart_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, chart_qn, rel_path, SKIP_ONE, 0, props); + free(chart_qn); + + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); + const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; + if (file_node && chart_id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, chart_id, "DEFINES", "{}"); + } + free(file_qn); + return chart_id > 0 ? chart_id : 0; +} + static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, k8s_local_deps_t *local_deps) { cbm_helm_chart_t hc; @@ -2781,24 +2815,14 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con } const char *cname = hc.chart_name[0] ? hc.chart_name : k8s_basename(rel_path); - char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); - if (!chart_qn) { + int64_t chart_id = k8s_chart_upsert(ctx, rel_path, cname, "{\"source\":\"helm\"}"); + if (chart_id <= 0) { cbm_helm_chart_free(&hc); return; } - int64_t chart_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", cname, chart_qn, rel_path, SKIP_ONE, - 0, "{\"source\":\"helm\"}"); - free(chart_qn); - - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); - const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; - if (file_node && chart_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, chart_id, "DEFINES", "{}"); - } - free(file_qn); int dep_edges = 0; - for (int i = 0; i < hc.dep_count && chart_id > 0; i++) { + for (int i = 0; i < hc.dep_count; i++) { const cbm_helm_dep_t *dep = &hc.deps[i]; if (k8s_is_file_repository(dep->repository)) { k8s_local_deps_push(local_deps, chart_id, rel_path, ctx->project_name, dep); @@ -2812,6 +2836,106 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con cbm_log_info("pass.k8s.helm", "file", rel_path, "deps", itoa_k8s(dep_edges)); } +/* ── werf chart handler ──────────────────────────────────────────── */ + +/* werf (werf.io) deploys the Helm chart in the `.helm` directory beside its + * werf.yaml; that chart needs no Chart.yaml, werf names it after its project. + * A `.helm` with a Chart.yaml is an ordinary chart. A custom + * deploy.helmChartDir is not followed. */ +static bool is_werf_file(const char *base) { + return strcmp(base, "werf.yaml") == 0 || strcmp(base, "werf.yml") == 0; +} + +bool cbm_werf_helm_dir(const char *repo_path, const char *dir) { + const char *base = k8s_basename(dir); + if (!repo_path || strcmp(base, ".helm") != 0) { + return false; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", repo_path, dir); + if (!cbm_is_dir(path)) { + return false; + } + static const char *const charts[] = {"Chart.yaml", "Chart.yml"}; + for (size_t i = 0; i < sizeof(charts) / sizeof(charts[0]); i++) { + snprintf(path, sizeof(path), "%s/%s/%s", repo_path, dir, charts[i]); + if (cbm_file_exists(path)) { + return false; + } + } + size_t parent = (size_t)(base - dir); + static const char *const werfs[] = {"werf.yaml", "werf.yml"}; + for (size_t i = 0; i < sizeof(werfs) / sizeof(werfs[0]); i++) { + snprintf(path, sizeof(path), "%s/%.*s%s", repo_path, (int)parent, dir, werfs[i]); + if (cbm_file_exists(path)) { + return true; + } + } + return false; +} + +/* The literal top-level `project:` of a werf.yaml's first document into out, + * "" when there is none or it is templated. */ +static void k8s_werf_project(const char *src, char *out, size_t out_sz) { + out[0] = '\0'; + bool seen_key = false; + for (const char *p = src; p && *p;) { + const char *eol = strchr(p, '\n'); + size_t len = eol ? (size_t)(eol - p) : strlen(p); + if (len >= K8S_SLEN("---") && strncmp(p, "---", K8S_SLEN("---")) == 0 && seen_key) { + return; + } + if (isalpha((unsigned char)p[0])) { + seen_key = true; + } + if (len > K8S_SLEN("project:") && strncmp(p, "project:", K8S_SLEN("project:")) == 0) { + /* the value: [b, e) of the line, a comment and quotes removed */ + size_t b = K8S_SLEN("project:"); + size_t e = len; + while (b < e && (p[b] == ' ' || p[b] == '\t')) { + b++; + } + for (size_t i = b; i < e; i++) { + if (p[i] == '#') { + e = i; + } + } + while (e > b && isspace((unsigned char)p[e - SKIP_ONE])) { + e--; + } + if (e >= b + PAIR_LEN && (p[b] == '"' || p[b] == '\'') && p[e - SKIP_ONE] == p[b]) { + b++; + e--; + } + if (e > b && e - b < out_sz && !memchr(p + b, '{', e - b)) { + memcpy(out, p + b, e - b); + out[e - b] = '\0'; + } + return; + } + p = eol ? eol + SKIP_ONE : NULL; + } +} + +/* A werf.yaml beside a `.helm` chart dir: a Chart named after its project + * {source: werf}, defined by werf.yaml, and the chart dir into `chart_dir`. + * False when it has no such chart. */ +static bool handle_werf_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, + char *chart_dir, size_t chart_dir_sz) { + char dir[CBM_SZ_1K]; + k8s_dirname(rel_path, dir, sizeof(dir)); + snprintf(chart_dir, chart_dir_sz, "%s%s.helm", dir, dir[0] ? "/" : ""); + if (!cbm_werf_helm_dir(ctx->repo_path, chart_dir)) { + return false; + } + char project[CBM_HELM_NAME_MAX]; + k8s_werf_project(source, project, sizeof(project)); + const char *cname = project[0] ? project : k8s_basename(rel_path); + int64_t chart_id = k8s_chart_upsert(ctx, rel_path, cname, "{\"source\":\"werf\"}"); + cbm_log_info("pass.k8s.werf", "file", rel_path, "chart", cname); + return chart_id > 0; +} + /* ── Dependency-manifest handler (go.mod / requirements.txt) ──────── */ static bool is_gomod_file(const char *base) { @@ -2969,7 +3093,7 @@ static void handle_dep_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c * the sequential loop did. Sequentially the pass re-read and re-parsed every * YAML manifest on one thread -- 1.1 s of the Go corpus (6,275 manifests; * profile + waste sanitizer, 2026-09-17). */ -typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_MANIFEST, K8S_KUSTOMIZE } k8s_kind_t; +typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_WERF, K8S_MANIFEST, K8S_KUSTOMIZE } k8s_kind_t; typedef struct { k8s_kind_t kind; @@ -3153,6 +3277,8 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { } if (is_helm_chart_file(base)) { out->kind = K8S_HELM; + } else if (is_werf_file(base)) { + out->kind = K8S_WERF; } else if (cbm_is_k8s_manifest(base, out->source)) { /* Always re-extract with CBM_LANG_K8S regardless of any cached * result: cached results were produced during the parallel YAML @@ -3610,7 +3736,8 @@ static void k8s_deploys_edge(k8s_dep_link_t *l, int64_t src, int64_t tgt, const } } -/* The Chart node of chart dir `dir`, 0 when it is none. */ +/* The Chart node of chart dir `dir`, 0 when it is none: its Chart.yaml's, or + * for a werf `.helm` dir the one its werf.yaml defines. */ static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { static const char *const names[] = {"Chart.yaml", "Chart.yml"}; for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { @@ -3621,6 +3748,19 @@ static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { return id; } } + const char *base = k8s_basename(dir); + if (strcmp(base, ".helm") != 0) { + return 0; + } + static const char *const werfs[] = {"werf.yaml", "werf.yml"}; + for (size_t i = 0; i < sizeof(werfs) / sizeof(werfs[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%.*s%s", (int)(base - dir), dir, werfs[i]); + int64_t id = k8s_first_id(l->charts, path); + if (id > 0) { + return id; + } + } return 0; } @@ -3661,8 +3801,10 @@ static bool k8s_chart_values_file(const char *rel, const char *dir) { /* A chart deploys what its templates render: Chart -DEPLOYS {via: chart}-> * each Resource of /templates/ -- its nearest enclosing chart, so a - * subchart's objects are the subchart's. Its values files (values.yaml, - * values-.yaml beside Chart.yaml) CONFIGURE {strategy: helm_values} it. */ + * subchart's objects are the subchart's -- and each CRD of /crds/, + * which Helm installs first ({via: chart, phase: crds}). Its values files + * (values.yaml, values-.yaml beside Chart.yaml) CONFIGURE {strategy: + * helm_values} it. */ static void k8s_link_chart_contents(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { if (!b->charts || cbm_ht_count(b->charts) == 0) { return; @@ -3691,9 +3833,14 @@ static void k8s_link_chart_contents(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { } if (k == 0) { char tpl[CBM_SZ_1K]; + char crd[CBM_SZ_1K]; snprintf(tpl, sizeof(tpl), "%s%stemplates/", dir, dir[0] ? "/" : ""); + snprintf(crd, sizeof(crd), "%s%scrds/", dir, dir[0] ? "/" : ""); if (strncmp(fp, tpl, strlen(tpl)) == 0) { k8s_deploys_edge(&l, chart, nodes[i]->id, "{\"via\":\"chart\"}"); + } else if (strncmp(fp, crd, strlen(crd)) == 0) { + k8s_deploys_edge(&l, chart, nodes[i]->id, + "{\"via\":\"chart\",\"phase\":\"crds\"}"); } } else if (k8s_chart_values_file(fp, dir)) { cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, chart, "CONFIGURES", @@ -3709,19 +3856,1039 @@ static void k8s_link_chart_contents(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { } } -/* Each Image a chart's objects name literally (their USAGE {ref: image}, after +/* ── Helm template images (PHASE 2 P2.5) ───────────────────────────── + * + * A chart template's container image is a placeholder in the skeleton + * (`<.Values.image.repository>`): it is rendered here (helm_template.c) from + * the chart's own values.yaml, its Chart.yaml (.Chart) and the defines of its + * chart family -- what `helm template` writes with the chart's defaults. The + * object then USES the Image {ref: image, default_of: the values keys read}, + * an image nothing else in the repository changes but a release can. A + * parent chart that changes it for its subchart (its `:` values, its + * `global:`) DEPLOYS the Image it makes {via: chart, ref: image, + * override_of}. A value that does not render (helm_template.h: UNKNOWN) or + * is no image reference names none. */ + +typedef struct { + int64_t node; + char *file; + char *container; /* NULL: unnamed */ + int start_line; + int end_line; +} k8s_timg_t; + +typedef struct { + char *dir; + int64_t chart; + cbm_helm_chart_meta_t meta; + yyjson_mut_val *chart_yaml; /* NULL: none */ + yyjson_mut_val *values; /* NULL: values.yaml does not parse (nothing renders) */ +} k8s_hchart_t; + +/* A template file of a chart family (its defines). */ +typedef struct { + char *file; + char *family; +} k8s_htpl_t; + +/* One rendered image of an object (or a chart / release). */ +typedef struct { + int64_t src; + int64_t img; + const char *ref; + const char *container; + const char *keys; /* JSON array; NULL: none */ +} k8s_himg_t; + +typedef struct { + k8s_himg_t *v; + int n; + int cap; +} k8s_himgs_t; + +typedef struct { + yyjson_mut_doc *doc; + CBMArena a; + CBMHashTable *charts; /* chart dir -> k8s_hchart_t* */ + CBMHashTable *defines; /* family dir -> cbm_helm_defines_t* */ + k8s_htpl_t *tpls; + int ntpl; + bool tpls_read; + k8s_timg_t *t; + int nt; + int tcap; + k8s_himgs_t dflt; /* the containers' images with the defaults */ + int edges; +} k8s_helm_t; + +/* The templated images of the facts, kept past them. */ +static void k8s_helm_collect(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f, k8s_helm_t *h) { + for (int i = 0; i < f->img_n; i++) { + const k8s_img_t *im = &f->imgs[i]; + const cbm_gbuf_node_t *n = + strchr(im->image, '<') ? cbm_gbuf_find_by_id(ctx->gbuf, im->node) : NULL; + if (!n || !n->file_path || im->start_line <= 0) { + continue; + } + if (!k8s_grow((void **)&h->t, &h->tcap, h->nt, sizeof(*h->t))) { + return; + } + h->t[h->nt++] = (k8s_timg_t){im->node, cbm_arena_strdup(&h->a, n->file_path), + im->container ? cbm_arena_strdup(&h->a, im->container) : NULL, + im->start_line, im->end_line}; + } +} + +static char *k8s_repo_read(cbm_pipeline_ctx_t *ctx, const char *rel, int *len) { + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + return k8s_read_file(path, len); +} + +/* A file of `dir` (one of `names`), parsed as values; *found when one exists. */ +static yyjson_mut_val *k8s_helm_yaml(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const char *dir, + const char *const *names, bool *found) { + *found = false; + for (int i = 0; names[i]; i++) { + char rel[CBM_SZ_1K]; + snprintf(rel, sizeof(rel), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + int len = 0; + char *src = k8s_repo_read(ctx, rel, &len); + if (!src) { + continue; + } + *found = true; + yyjson_mut_val *v = cbm_helm_values_parse(h->doc, src, (size_t)len); + cbm_free(CBM_MEM_CLASS_OTHER, src); + return v; + } + return NULL; +} + +static const char *k8s_helm_meta_str(yyjson_mut_val *obj, const char *key) { + yyjson_mut_val *v = yyjson_mut_is_obj(obj) ? yyjson_mut_obj_get(obj, key) : NULL; + return yyjson_mut_is_str(v) ? yyjson_mut_get_str(v) : NULL; +} + +/* The chart of chart dir `dir` with its metadata and default values. */ +static k8s_hchart_t *k8s_helm_chart(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_dep_link_t *l, + const char *dir) { + k8s_hchart_t *c = cbm_ht_get(h->charts, dir); + if (c) { + return c; + } + c = cbm_arena_calloc(&h->a, sizeof(*c)); + if (!c) { + return NULL; + } + c->dir = cbm_arena_strdup(&h->a, dir); + c->chart = k8s_chart_node(l, dir); + static const char *const CHART[] = {"Chart.yaml", "Chart.yml", NULL}; + static const char *const VALUES[] = {"values.yaml", "values.yml", NULL}; + bool found = false; + c->chart_yaml = k8s_helm_yaml(ctx, h, dir, CHART, &found); + yyjson_mut_val *vals = k8s_helm_yaml(ctx, h, dir, VALUES, &found); + c->values = found ? vals : yyjson_mut_obj(h->doc); + const cbm_gbuf_node_t *cn = c->chart > 0 ? cbm_gbuf_find_by_id(ctx->gbuf, c->chart) : NULL; + const char *name = k8s_helm_meta_str(c->chart_yaml, "name"); + c->meta.name = name ? name : cn && cn->name ? cbm_arena_strdup(&h->a, cn->name) : NULL; + c->meta.version = k8s_helm_meta_str(c->chart_yaml, "version"); + c->meta.app_version = k8s_helm_meta_str(c->chart_yaml, "appVersion"); + cbm_ht_set(h->charts, c->dir, c); + return c; +} + +static bool k8s_helm_tpl_ext(const char *p) { + static const char *const EXT[] = {".yaml", ".yml", ".tpl", ".gotmpl", ".txt", ".json"}; + for (size_t i = 0; i < sizeof(EXT) / sizeof(EXT[0]); i++) { + if (k8s_has_suffix(p, EXT[i])) { + return true; + } + } + return false; +} + +/* The defines of chart family `family`: every template (/templates/) + * of its charts, read from disk. */ +static const cbm_helm_defines_t *k8s_helm_defines(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, + k8s_helm_t *h, const char *family) { + cbm_helm_defines_t *d = cbm_ht_get(h->defines, family); + if (d) { + return d; + } + if (!h->tpls_read) { + h->tpls_read = true; + const cbm_gbuf_node_t **files = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "File", &files, &n) == 0 && n > 0) { + h->tpls = cbm_arena_alloc(&h->a, (size_t)n * sizeof(*h->tpls)); + for (int i = 0; h->tpls && i < n; i++) { + const char *fp = files[i]->file_path; + char dir[CBM_SZ_1K]; + char fam[CBM_SZ_1K]; + char tdir[CBM_SZ_1K]; + if (!fp || !k8s_helm_tpl_ext(fp) || !k8s_chart_of(b, fp, dir, sizeof(dir))) { + continue; + } + snprintf(tdir, sizeof(tdir), "%s%stemplates/", dir, dir[0] ? "/" : ""); + if (strncmp(fp, tdir, strlen(tdir)) != 0) { + continue; + } + k8s_chart_family(b, dir, fam, sizeof(fam)); + h->tpls[h->ntpl++] = + (k8s_htpl_t){cbm_arena_strdup(&h->a, fp), cbm_arena_strdup(&h->a, fam)}; + } + } + } + d = cbm_helm_defines_new(); + if (!d) { + return NULL; + } + for (int i = 0; i < h->ntpl; i++) { + if (strcmp(h->tpls[i].family, family) != 0) { + continue; + } + int len = 0; + char *src = k8s_repo_read(ctx, h->tpls[i].file, &len); + if (src) { + cbm_helm_defines_add(d, h->tpls[i].file, src, (size_t)len); + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + } + cbm_ht_set(h->defines, cbm_arena_strdup(&h->a, family), d); + return d; +} + +static void k8s_helm_free_defines(const char *key, void *value, void *ud) { + (void)key; + (void)ud; + cbm_helm_defines_free(value); +} + +static int k8s_timg_cmp(const void *x, const void *y) { + const k8s_timg_t *a = x; + const k8s_timg_t *b = y; + int c = strcmp(a->file, b->file); + return c ? c : a->start_line - b->start_line; +} + +/* Does template line `line` (1-based) of src hold an `image:` key? */ +static bool k8s_helm_image_line(const char *src, size_t len, const uint32_t *starts, int line) { + size_t k = starts[line - SKIP_ONE]; + while (k < len && (src[k] == ' ' || src[k] == '\t')) { + k++; + } + if (k + SKIP_ONE < len && src[k] == '-' && src[k + SKIP_ONE] == ' ') { + k++; + while (k < len && src[k] == ' ') { + k++; + } + } + return k + strlen("image:") <= len && memcmp(src + k, "image:", strlen("image:")) == 0; +} + +/* The probes of a template's templated images [from, to): each `image:` + * line inside a container (both branches of an if / else among them). */ +static int k8s_helm_probes(k8s_helm_t *h, const k8s_timg_t *t, int from, int to, const char *src, + size_t len, cbm_helm_probe_t **out) { + int nl = SKIP_ONE; + for (size_t i = 0; i < len; i++) { + nl += src[i] == '\n'; + } + uint32_t *starts = cbm_arena_alloc(&h->a, (size_t)nl * sizeof(*starts)); + if (!starts) { + return 0; + } + int k = 0; + starts[k++] = 0; + for (size_t i = 0; i < len && k < nl; i++) { + if (src[i] == '\n') { + starts[k++] = (uint32_t)(i + SKIP_ONE); + } + } + int cap = 0; + for (int i = from; i < to; i++) { + cap += t[i].end_line - t[i].start_line + SKIP_ONE; + } + cbm_helm_probe_t *p = cap ? cbm_arena_calloc(&h->a, (size_t)cap * sizeof(*p)) : NULL; + int np = 0; + for (int i = from; p && i < to; i++) { + for (int line = t[i].start_line; line <= t[i].end_line && line <= k; line++) { + bool dup = false; + for (int j = 0; j < np && !dup; j++) { + dup = p[j].line == line; + } + if (!dup && k8s_helm_image_line(src, len, starts, line)) { + p[np++].line = line; + } + } + } + *out = p; + return np; +} + +static yyjson_mut_val *k8s_helm_dig(yyjson_mut_val *v, const char *path) { + const char *p = path; + while (v && *p) { + const char *dot = strchr(p, '.'); + size_t kl = dot ? (size_t)(dot - p) : strlen(p); + v = yyjson_mut_is_obj(v) ? yyjson_mut_obj_getn(v, p, kl) : NULL; + p += kl + (dot ? SKIP_ONE : 0); + } + return v; +} + +/* Is values key path `key` (dot-joined, `\.` escaped) set in `v`? */ +static bool k8s_helm_sets(yyjson_mut_val *v, const char *key) { + char part[CBM_SZ_512]; + const char *p = key; + while (v && *p) { + size_t k = 0; + while (*p && *p != '.' && *p != '[' && k + SKIP_ONE < sizeof(part)) { + if (*p == '\\' && p[1]) { + p++; + } + part[k++] = *p++; + } + part[k] = '\0'; + v = yyjson_mut_is_obj(v) ? yyjson_mut_obj_get(v, part) : NULL; + while (v && *p == '[') { + long idx = strtol(p + SKIP_ONE, (char **)&p, 10); + p += *p == ']' ? SKIP_ONE : 0; + v = yyjson_mut_is_arr(v) ? yyjson_mut_arr_get(v, (size_t)idx) : NULL; + } + if (*p == '.') { + p++; + } + } + return v != NULL; +} + +/* The values keys of JSON array `keys` that `overlay` sets (relative to + * the chart), prefixed by `prefix` -- as a JSON array; NULL: none. */ +static const char *k8s_helm_overridden(k8s_helm_t *h, const char *keys, yyjson_mut_val *overlay, + yyjson_mut_val *globals, const char *prefix) { + if (!keys) { + return NULL; + } + yyjson_doc *doc = yyjson_read(keys, strlen(keys), 0); + yyjson_val *arr = doc ? yyjson_doc_get_root(doc) : NULL; + const char *out = NULL; + if (yyjson_is_arr(arr)) { + size_t idx, max; + yyjson_val *k; + yyjson_arr_foreach(arr, idx, max, k) { + const char *key = yyjson_get_str(k); + bool global = key && strncmp(key, "global.", strlen("global.")) == 0; + bool set = key && (global ? k8s_helm_sets(globals, key + strlen("global.")) + : k8s_helm_sets(overlay, key)); + if (!set) { + continue; + } + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), key); + out = cbm_arena_sprintf(&h->a, "%s%s\"%s%s%s\"", out ? out : "[", out ? "," : "", + global || !prefix ? "" : prefix, global || !prefix ? "" : ".", + esc); + } + } + if (doc) { + yyjson_doc_free(doc); + } + return out ? cbm_arena_sprintf(&h->a, "%s]", out) : NULL; +} + +static void k8s_himg_push(k8s_himgs_t *l, k8s_himg_t x) { + for (int i = 0; i < l->n; i++) { + if (l->v[i].src == x.src && l->v[i].img == x.img) { + return; /* the first (in file order) of the same Image stands for it */ + } + } + if (k8s_grow((void **)&l->v, &l->cap, l->n, sizeof(*l->v))) { + l->v[l->n++] = x; + } +} + +/* Does `src` USE Image `img` already by a literal reference? */ +static bool k8s_helm_literal_use(cbm_pipeline_ctx_t *ctx, int64_t src, int64_t img) { + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, src, "USAGE", &es, &n) != 0) { + return false; + } + for (int i = 0; i < n; i++) { + if (es[i]->target_id == img && es[i]->properties_json && + strstr(es[i]->properties_json, "\"ref\":\"image\"")) { + return true; + } + } + return false; +} + +/* The captures of probes within [first, last] lines, as images of `src`. */ +static void k8s_helm_take(cbm_pipeline_ctx_t *ctx, const cbm_helm_probe_t *p, int np, int first, + int last, int64_t src, const char *container, k8s_himgs_t *out) { + for (int i = 0; i < np; i++) { + if (p[i].line < first || p[i].line > last) { + continue; + } + for (int c = 0; c < p[i].n; c++) { + const char *ref = p[i].image[c]; + int64_t img = ref && cbm_image_ref_valid(ref) ? cbm_pipeline_image_node(ctx, ref) : 0; + if (img > 0) { + k8s_himg_push(out, (k8s_himg_t){src, img, ref, container, p[i].keys[c]}); + } + } + } +} + +/* The images of templated containers [from, to) of one template, rendered + * with `values` (NULL: the chart's own). */ +static void k8s_helm_render(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, + const k8s_hchart_t *c, const k8s_timg_t *t, int from, int to, + yyjson_mut_val *values, const char *src, size_t len, + cbm_helm_probe_t *probes, int np) { + char fam[CBM_SZ_1K]; + k8s_chart_family(b, c->dir, fam, sizeof(fam)); + const cbm_helm_defines_t *defs = k8s_helm_defines(ctx, b, h, fam); + cbm_helm_render(&h->a, t[from].file, src, len, values ? values : c->values, &c->meta, defs, + probes, np); + (void)to; +} + +static void k8s_helm_emit_uses(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const k8s_himgs_t *l) { + for (int i = 0; i < l->n; i++) { + const k8s_himg_t *x = &l->v[i]; + if (k8s_helm_literal_use(ctx, x->src, x->img)) { + continue; /* a literal reference of the object wins */ + } + char esc[CBM_SZ_1K]; + char cesc[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), x->ref); + cbm_json_escape(cesc, (int)sizeof(cesc), x->container ? x->container : "-"); + const char *props = cbm_arena_sprintf( + &h->a, "{\"ref\":\"image\",\"image\":\"%s\",\"container\":\"%s\"%s%s}", esc, cesc, + x->keys ? ",\"default_of\":" : "", x->keys ? x->keys : ""); + if (props) { + cbm_gbuf_insert_edge(ctx->gbuf, x->src, x->img, "USAGE", props); + h->edges++; + } + } +} + +/* Chart `from` -DEPLOYS {via, ref: image, image, override_of}-> the images an + * overlay changed: rendered there, absent from the defaults' images. */ +static void k8s_helm_emit_overrides(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t from, + const char *via, const k8s_himgs_t *dflt, + const k8s_himgs_t *got, yyjson_mut_val *overlay, + yyjson_mut_val *globals, const char *prefix, + k8s_himgs_t *emitted) { + for (int i = 0; i < got->n; i++) { + const k8s_himg_t *x = &got->v[i]; + bool same = false; + for (int j = 0; j < dflt->n && !same; j++) { + same = dflt->v[j].src == x->src && dflt->v[j].img == x->img && + strcmp(dflt->v[j].ref, x->ref) == 0; + } + const char *ovr = k8s_helm_overridden(h, x->keys, overlay, globals, prefix); + if (same || !ovr || from <= 0) { + continue; + } + k8s_himg_t e = {from, x->img, x->ref, NULL, ovr}; + int before = emitted->n; + k8s_himg_push(emitted, e); + if (emitted->n == before) { + continue; + } + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), x->ref); + const char *props = cbm_arena_sprintf( + &h->a, "{\"via\":\"%s\",\"ref\":\"image\",\"image\":\"%s\",\"override_of\":%s}", via, + esc, ovr); + if (props) { + cbm_gbuf_insert_edge(ctx->gbuf, from, x->img, "DEPLOYS", props); + h->edges++; + } + } +} + +/* The chart dirs enclosing chart dir `dir`, nearest first (arena). */ +static int k8s_helm_ancestors(k8s_helm_t *h, k8s_bundles_t *b, const char *dir, const char ***out) { + int n = 0; + int cap = 0; + const char **v = NULL; + char cur[CBM_SZ_1K]; + snprintf(cur, sizeof(cur), "%s", dir); + while (cur[0]) { + char up[CBM_SZ_1K]; + if (!k8s_chart_of(b, cur, up, sizeof(up))) { + break; + } + if (n == cap) { + cap = cap ? cap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(&h->a, (size_t)cap * sizeof(*g)); + if (!g) { + break; + } + if (n) { + memcpy(g, v, (size_t)n * sizeof(*g)); + } + v = g; + } + v[n++] = cbm_arena_strdup(&h->a, up); + snprintf(cur, sizeof(cur), "%s", up); + } + *out = v; + return n; +} + +/* The dependency of `parent`'s Chart.yaml naming `sub` (by name), NULL: none. */ +static yyjson_mut_val *k8s_helm_dep(const k8s_hchart_t *parent, const k8s_hchart_t *sub) { + yyjson_mut_val *deps = yyjson_mut_is_obj(parent->chart_yaml) + ? yyjson_mut_obj_get(parent->chart_yaml, "dependencies") + : NULL; + if (yyjson_mut_is_arr(deps) && sub->meta.name) { + size_t idx, max; + yyjson_mut_val *d; + yyjson_mut_arr_foreach(deps, idx, max, d) { + const char *name = k8s_helm_meta_str(d, "name"); + if (name && strcmp(name, sub->meta.name) == 0) { + return d; + } + } + } + return NULL; +} + +/* Is the subchart a dependency disables in the parent's values `vals` (its + * `condition`: the first of its comma-separated paths that holds a bool)? */ +static bool k8s_helm_disabled(k8s_helm_t *h, yyjson_mut_val *dep, yyjson_mut_val *vals) { + const char *cond = dep ? k8s_helm_meta_str(dep, "condition") : NULL; + for (const char *p = cond; p && *p;) { + const char *comma = strchr(p, ','); + size_t l = comma ? (size_t)(comma - p) : strlen(p); + char *path = cbm_arena_strndup(&h->a, p, l); + yyjson_mut_val *v = path ? k8s_helm_dig(vals, path) : NULL; + if (yyjson_mut_is_bool(v)) { + return !yyjson_mut_get_bool(v); + } + p += l + (comma ? SKIP_ONE : 0); + } + return false; +} + +/* The values chart dir `dir` renders with when chart dir `top` deploys it + * with values `top_vals` (NULL: top's defaults): down the charts between + * them, each subchart's defaults with its parent's `:` values and + * `global:` coalesced over them (Helm's coalesce; a subchart always has a + * `global`). *path gets dir's key path from top ("" for top itself). NULL: + * a dependency condition disables it, or a values file does not parse. */ +static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, + k8s_dep_link_t *l, const char *dir, const char *top, + yyjson_mut_val *top_vals, const char **path) { + const char **anc = NULL; + int na = k8s_helm_ancestors(h, b, dir, &anc); + int ti = -SKIP_ONE; /* top's index among the ancestors; na: dir itself */ + if (strcmp(dir, top) == 0) { + ti = na; + } else { + for (int i = 0; i < na && ti < 0; i++) { + ti = strcmp(anc[i], top) == 0 ? i : ti; + } + } + *path = ""; + if (ti < 0) { + return NULL; + } + k8s_hchart_t *cur_chart = k8s_helm_chart(ctx, h, l, top); + if (!cur_chart || !cur_chart->values) { + return NULL; + } + yyjson_mut_val *cur = top_vals ? cbm_helm_values_coalesce(h->doc, cur_chart->values, top_vals) + : cur_chart->values; + for (int i = (ti == na ? na : ti) - SKIP_ONE; i >= -SKIP_ONE && ti != na; i--) { + k8s_hchart_t *child = k8s_helm_chart(ctx, h, l, i >= 0 ? anc[i] : dir); + if (!child || !child->values) { + return NULL; + } + yyjson_mut_val *dep = k8s_helm_dep(cur_chart, child); + if (k8s_helm_disabled(h, dep, cur)) { + return NULL; + } + const char *alias = dep ? k8s_helm_meta_str(dep, "alias") : NULL; + alias = alias ? alias : child->meta.name; + if (!alias) { + return NULL; + } + *path = (*path)[0] ? cbm_arena_sprintf(&h->a, "%s.%s", *path, alias) + : cbm_arena_strdup(&h->a, alias); + yyjson_mut_val *sub = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, alias) : NULL; + yyjson_mut_val *next = cbm_helm_values_coalesce(h->doc, child->values, sub); + yyjson_mut_val *pg = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, "global") : NULL; + yyjson_mut_val *cg = next ? yyjson_mut_obj_get(next, "global") : NULL; + yyjson_mut_val *g = cbm_helm_values_coalesce(h->doc, cg, pg); /* the parent's win */ + if (!next || !g) { + return NULL; + } + (void)yyjson_mut_obj_remove_key(next, "global"); + yyjson_mut_obj_add(next, yyjson_mut_strcpy(h->doc, "global"), g); + cur = next; + cur_chart = child; + } + return cur; +} + +/* Render the templated containers [from, to) of one template with `vals` + * and take each container's images. */ +static void k8s_helm_images_of(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, + const k8s_hchart_t *c, int from, int to, yyjson_mut_val *vals, + const char *src, size_t len, cbm_helm_probe_t *probes, int np, + k8s_himgs_t *out) { + k8s_helm_render(ctx, h, b, c, h->t, from, to, vals, src, len, probes, np); + for (int i = from; i < to; i++) { + k8s_helm_take(ctx, probes, np, h->t[i].start_line, h->t[i].end_line, h->t[i].node, + h->t[i].container, out); + } +} + +/* Every templated container's images with its chart family's defaults -- + * what `helm template ` renders. */ +static void k8s_helm_link_defaults(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h) { + if (h->nt == 0 || !ctx->repo_path) { + return; + } + qsort(h->t, (size_t)h->nt, sizeof(*h->t), k8s_timg_cmp); + k8s_dep_link_t l = {.ctx = ctx, .charts = k8s_index_by_file(ctx, "Chart")}; + k8s_himgs_t uses = {0}; + for (int from = 0; from < h->nt;) { + int to = from + SKIP_ONE; + while (to < h->nt && strcmp(h->t[to].file, h->t[from].file) == 0) { + to++; + } + char dir[CBM_SZ_1K]; + char fam[CBM_SZ_1K]; + k8s_hchart_t *c = k8s_chart_of(b, h->t[from].file, dir, sizeof(dir)) + ? k8s_helm_chart(ctx, h, &l, dir) + : NULL; + const char *path = NULL; + yyjson_mut_val *vals = NULL; + if (c) { + k8s_chart_family(b, c->dir, fam, sizeof(fam)); + vals = k8s_helm_context(ctx, h, b, &l, c->dir, fam, NULL, &path); + } + int len = 0; + char *src = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; + cbm_helm_probe_t *probes = NULL; + int np = src ? k8s_helm_probes(h, h->t, from, to, src, (size_t)len, &probes) : 0; + if (np > 0) { + k8s_helm_images_of(ctx, h, b, c, from, to, vals, src, (size_t)len, probes, np, &uses); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + from = to; + } + k8s_helm_emit_uses(ctx, h, &uses); + h->dflt = uses; /* the release views compare against these */ + k8s_free_index(l.charts); +} + +/* ── Release images ─────────────────────────────────────────────────── + * + * A release (a Flux HelmRelease's spec.values, a kustomize helmCharts + * entry's valuesFile + valuesInline, a Terraform helm_release's values + + * set) renders its chart with its values over the defaults. Of a chart in + * the repository the templates render again: each image that differs from + * the defaults' is the release's own -- release -DEPLOYS {via, ref: image, + * image, override_of: the keys it set that the image read}-> Image. A chart + * outside the repository has no templates here: its images are read from the + * release values by convention -- a mapping with `repository` (and `tag` / + * `digest` / `registry`) whose reference names a registry host; one without + * a host names none (no host is invented) -- {inferred: values, values_key}. */ + +/* The chart dir of local Chart node `cn` (a werf chart: its `.helm`). */ +static bool k8s_helm_chart_dir(const cbm_gbuf_node_t *cn, char *out, size_t sz) { + if (!cn->file_path) { + return false; + } + k8s_dirname(cn->file_path, out, sz); + const char *base = k8s_basename(cn->file_path); + if (strcmp(base, "werf.yaml") == 0 || strcmp(base, "werf.yml") == 0) { + size_t l = strlen(out); + snprintf(out + l, sz - l, "%s.helm", l ? "/" : ""); + } + return true; +} + +/* A local chart's images rendered with release values `rv`, against the + * defaults. */ +static void k8s_helm_release_local(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h, + k8s_dep_link_t *l, int64_t src, const char *via, const char *top, + yyjson_mut_val *rv, k8s_himgs_t *emitted) { + size_t tl = strlen(top); + for (int from = 0; from < h->nt;) { + int to = from + SKIP_ONE; + while (to < h->nt && strcmp(h->t[to].file, h->t[from].file) == 0) { + to++; + } + char dir[CBM_SZ_1K]; + bool in = + k8s_chart_of(b, h->t[from].file, dir, sizeof(dir)) && + (strcmp(dir, top) == 0 || (tl == 0 || (strncmp(dir, top, tl) == 0 && dir[tl] == '/'))); + k8s_hchart_t *c = in ? k8s_helm_chart(ctx, h, l, dir) : NULL; + const char *path = NULL; + yyjson_mut_val *vals = c ? k8s_helm_context(ctx, h, b, l, dir, top, rv, &path) : NULL; + int len = 0; + char *srcs = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; + cbm_helm_probe_t *probes = NULL; + int np = srcs ? k8s_helm_probes(h, h->t, from, to, srcs, (size_t)len, &probes) : 0; + if (np > 0) { + k8s_himgs_t got = {0}; + k8s_helm_images_of(ctx, h, b, c, from, to, vals, srcs, (size_t)len, probes, np, &got); + yyjson_mut_val *overlay = path && path[0] ? k8s_helm_dig(rv, path) : rv; + yyjson_mut_val *globals = yyjson_mut_obj_get(rv, "global"); + k8s_helm_emit_overrides(ctx, h, src, via, &h->dflt, &got, overlay, globals, + path && path[0] ? path : NULL, emitted); + cbm_free(CBM_MEM_CLASS_OTHER, got.v); + } + cbm_free(CBM_MEM_CLASS_OTHER, srcs); + from = to; + } +} + +/* Does image reference `ref` name a registry host (its first path part has + * a `.` or `:`, or is localhost)? */ +static bool k8s_helm_has_host(const char *ref) { + const char *slash = strchr(ref, '/'); + if (!slash) { + return false; + } + size_t fl = (size_t)(slash - ref); + return memchr(ref, '.', fl) || memchr(ref, ':', fl) || + (fl == strlen("localhost") && strncmp(ref, "localhost", fl) == 0); +} + +/* The images release values `v` name by convention (see above). */ +static void k8s_helm_convention(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t src, + const char *via, yyjson_mut_val *v, const char *path, + k8s_himgs_t *emitted) { + size_t idx, max; + if (yyjson_mut_is_arr(v)) { + yyjson_mut_val *it; + yyjson_mut_arr_foreach(v, idx, max, it) { + k8s_helm_convention(ctx, h, src, via, it, + cbm_arena_sprintf(&h->a, "%s[%zu]", path ? path : "", idx), + emitted); + } + return; + } + if (!yyjson_mut_is_obj(v)) { + return; + } + yyjson_mut_val *repo = yyjson_mut_obj_get(v, "repository"); + if (yyjson_mut_is_str(repo) && yyjson_mut_get_len(repo) > 0) { + char reg[CBM_SZ_512] = ""; + char tag[CBM_SZ_256] = ""; + char dig[CBM_SZ_256] = ""; + yyjson_mut_val *r = yyjson_mut_obj_get(v, "registry"); + yyjson_mut_val *tg = yyjson_mut_obj_get(v, "tag"); + yyjson_mut_val *dg = yyjson_mut_obj_get(v, "digest"); + bool ok = (!r || yyjson_mut_is_null(r) || cbm_helm_scalar_text(r, reg, sizeof(reg))) && + (!tg || yyjson_mut_is_null(tg) || cbm_helm_scalar_text(tg, tag, sizeof(tag))) && + (!dg || yyjson_mut_is_null(dg) || cbm_helm_scalar_text(dg, dig, sizeof(dig))); + const char *ref = ok ? cbm_arena_sprintf(&h->a, "%s%s%s%s%s", reg, reg[0] ? "/" : "", + yyjson_mut_get_str(repo), + dig[0] ? "@" + : tag[0] ? ":" + : "", + dig[0] ? dig : tag) + : NULL; + int64_t img = ref && k8s_helm_has_host(ref) && cbm_image_ref_valid(ref) + ? cbm_pipeline_image_node(ctx, ref) + : 0; + int before = emitted->n; + if (img > 0) { + k8s_himg_push(emitted, (k8s_himg_t){src, img, ref, NULL, NULL}); + } + if (emitted->n > before) { + char esc[CBM_SZ_1K]; + char kesc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), ref); + cbm_json_escape(kesc, (int)sizeof(kesc), path ? path : ""); + const char *props = cbm_arena_sprintf( + &h->a, + "{\"via\":\"%s\",\"ref\":\"image\",\"image\":\"%s\",\"inferred\":\"values\"," + "\"values_key\":\"%s\"}", + via, esc, kesc); + if (props) { + cbm_gbuf_insert_edge(ctx->gbuf, src, img, "DEPLOYS", props); + h->edges++; + } + } + } + yyjson_mut_val *k; + yyjson_mut_val *child; + yyjson_mut_obj_foreach(v, idx, max, k, child) { + if (!yyjson_mut_is_obj(child) && !yyjson_mut_is_arr(child)) { + continue; + } + const char *key = yyjson_mut_get_str(k); + const char *cp = path && path[0] ? cbm_arena_sprintf(&h->a, "%s.%s", path, key) + : cbm_arena_strdup(&h->a, key); + k8s_helm_convention(ctx, h, src, via, child, cp, emitted); + } +} + +/* One release of Chart node `chart` with values `rv`. */ +static void k8s_helm_release(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h, + k8s_dep_link_t *l, int64_t src, const char *via, int64_t chart, + yyjson_mut_val *rv) { + const cbm_gbuf_node_t *cn = chart > 0 ? cbm_gbuf_find_by_id(ctx->gbuf, chart) : NULL; + if (!cn || !yyjson_mut_is_obj(rv) || src <= 0) { + return; + } + k8s_himgs_t emitted = {0}; + char dir[CBM_SZ_1K]; + if (cbm_pipeline_chart_is_external(cn)) { + k8s_helm_convention(ctx, h, src, via, rv, NULL, &emitted); + } else if (k8s_helm_chart_dir(cn, dir, sizeof(dir)) && b->charts && + cbm_ht_get(b->charts, dir)) { + k8s_helm_release_local(ctx, b, h, l, src, via, dir, rv, &emitted); + } + cbm_free(CBM_MEM_CLASS_OTHER, emitted.v); +} + +/* A node's document (lines start_line..end_line of its file) as values. */ +static yyjson_mut_val *k8s_helm_node_yaml(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, + const cbm_gbuf_node_t *n) { + int len = 0; + char *src = n->file_path ? k8s_repo_read(ctx, n->file_path, &len) : NULL; + if (!src) { + return NULL; + } + size_t b = 0; + size_t e = (size_t)len; + int line = SKIP_ONE; + for (size_t i = 0; i < (size_t)len && n->start_line > 0; i++) { + if (src[i] != '\n') { + continue; + } + line++; + if (line == n->start_line) { + b = i + SKIP_ONE; + } + if (n->end_line > 0 && line == n->end_line + SKIP_ONE) { + e = i; + break; + } + } + yyjson_mut_val *v = b <= e ? cbm_helm_values_parse(h->doc, src + b, e - b) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, src); + return v; +} + +/* String property `key` of edge properties `props` into out; false: none. */ +static bool k8s_json_str(const char *props, const char *key, char *out, size_t sz) { + out[0] = '\0'; + yyjson_doc *d = props ? yyjson_read(props, strlen(props), 0) : NULL; + yyjson_val *v = d ? yyjson_obj_get(yyjson_doc_get_root(d), key) : NULL; + bool ok = yyjson_is_str(v); + if (ok) { + snprintf(out, sz, "%s", yyjson_get_str(v)); + } + if (d) { + yyjson_doc_free(d); + } + return ok; +} + +/* A kustomize helmCharts entry's values: valuesFile, then valuesInline over + * it; NULL when neither. */ +static yyjson_mut_val *k8s_helm_kustomize_values(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, + const cbm_gbuf_node_t *kust, const char *props) { + char chart[CBM_SZ_512] = ""; + char release[CBM_SZ_512] = ""; + char vfile[CBM_SZ_1K] = ""; + (void)k8s_json_str(props, "chart", chart, sizeof(chart)); + (void)k8s_json_str(props, "release", release, sizeof(release)); + (void)k8s_json_str(props, "values", vfile, sizeof(vfile)); + int len = 0; + char *src = kust->file_path ? k8s_repo_read(ctx, kust->file_path, &len) : NULL; + yyjson_mut_val *doc = src ? cbm_helm_values_parse(h->doc, src, (size_t)len) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, src); + yyjson_mut_val *charts = yyjson_mut_is_obj(doc) ? yyjson_mut_obj_get(doc, "helmCharts") : NULL; + yyjson_mut_val *inline_vals = NULL; + size_t idx, max; + yyjson_mut_val *c; + yyjson_mut_arr_foreach(charts, idx, max, c) { + const char *name = k8s_helm_meta_str(c, "name"); + const char *rel = k8s_helm_meta_str(c, "releaseName"); + if (name && strcmp(name, chart) == 0 && + (!release[0] || (rel && strcmp(rel, release) == 0))) { + inline_vals = yyjson_mut_obj_get(c, "valuesInline"); + break; + } + } + yyjson_mut_val *file_vals = NULL; + if (vfile[0]) { + int fl = 0; + char *fs = k8s_repo_read(ctx, vfile, &fl); + file_vals = fs ? cbm_helm_values_parse(h->doc, fs, (size_t)fl) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, fs); + } + if (!inline_vals && !file_vals) { + return NULL; + } + return cbm_helm_values_coalesce(h->doc, file_vals, inline_vals); +} + +/* Every release's images: HelmReleases and kustomize helmCharts from their + * DEPLOYS edges to a Chart, Terraform helm_releases from the context. */ +static void k8s_helm_link_releases(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm_t *h) { + k8s_dep_link_t l = {.ctx = ctx, .charts = k8s_index_by_file(ctx, "Chart")}; + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + if (ctx->repo_path && cbm_gbuf_find_edges_by_type(ctx->gbuf, "DEPLOYS", &es, &n) == 0) { + /* collected first: release edges are DEPLOYS too */ + typedef struct { + int64_t src; + int64_t tgt; + const char *props; + } k8s_rel_t; + k8s_rel_t *rels = n ? cbm_arena_alloc(&h->a, (size_t)n * sizeof(*rels)) : NULL; + int nr = 0; + for (int i = 0; rels && i < n; i++) { + const char *p = es[i]->properties_json; + if (p && + (strstr(p, "\"via\":\"helmrelease\"") || strstr(p, "\"via\":\"helmcharts\"")) && + !strstr(p, "\"ref\":")) { + rels[nr++] = + (k8s_rel_t){es[i]->source_id, es[i]->target_id, cbm_arena_strdup(&h->a, p)}; + } + } + for (int i = 0; i < nr; i++) { + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, rels[i].src); + if (!src) { + continue; + } + bool hr = strstr(rels[i].props, "\"via\":\"helmrelease\"") != NULL; + yyjson_mut_val *rv = NULL; + if (hr) { + yyjson_mut_val *docv = k8s_helm_node_yaml(ctx, h, src); + yyjson_mut_val *spec = + yyjson_mut_is_obj(docv) ? yyjson_mut_obj_get(docv, "spec") : NULL; + rv = yyjson_mut_is_obj(spec) ? yyjson_mut_obj_get(spec, "values") : NULL; + } else { + rv = k8s_helm_kustomize_values(ctx, h, src, rels[i].props); + } + k8s_helm_release(ctx, b, h, &l, rels[i].src, hr ? "helmrelease" : "helmcharts", + rels[i].tgt, rv); + } + } + for (int i = 0; i < ctx->tf_release_n; i++) { + cbm_tf_release_t *r = &ctx->tf_releases[i]; + if (r->values) { + yyjson_doc *jd = yyjson_read(r->values, strlen(r->values), 0); + yyjson_mut_val *rv = jd ? yyjson_val_mut_copy(h->doc, yyjson_doc_get_root(jd)) : NULL; + if (jd) { + yyjson_doc_free(jd); + } + k8s_helm_release(ctx, b, h, &l, r->src, "terraform", r->chart, rv); + cbm_free(CBM_MEM_CLASS_OTHER, r->values); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, ctx->tf_releases); + ctx->tf_releases = NULL; + ctx->tf_release_n = 0; + ctx->tf_release_cap = 0; + k8s_free_index(l.charts); +} + +static void k8s_helm_init(k8s_helm_t *h) { + *h = (k8s_helm_t){0}; + cbm_arena_init_lazy(&h->a, CBM_SZ_4K); + h->doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); + h->charts = cbm_ht_create(0); + h->defines = cbm_ht_create(0); +} + +static void k8s_helm_free(k8s_helm_t *h) { + if (h->defines) { + cbm_ht_foreach(h->defines, k8s_helm_free_defines, NULL); + cbm_ht_free(h->defines); + } + if (h->charts) { + cbm_ht_free(h->charts); + } + if (h->doc) { + yyjson_mut_doc_free(h->doc); + } + cbm_free(CBM_MEM_CLASS_OTHER, h->t); + cbm_free(CBM_MEM_CLASS_OTHER, h->dflt.v); + cbm_arena_destroy(&h->a); + if (h->edges > 0) { + cbm_log_info("pass.k8s.helm_images", "edges", itoa_k8s(h->edges)); + } + *h = (k8s_helm_t){0}; +} + +/* Each Image a chart's objects use (their USAGE {ref: image}, after * pass_docker.c moved references onto component Images): Chart -DEPLOYS - * {via: chart, ref: image}-> it. A templated image names no Image, and no - * values default is read. Edges are collected first: inserting while walking - * a chart's own DEPLOYS edges would grow the list being read. */ + * {via: chart, ref: image}-> it -- {default_of: the values keys} when every + * object of the chart uses it by a rendered default (the keys unioned), none + * when one names it literally. Edges are collected first: inserting while + * walking a chart's own DEPLOYS edges would grow the list being read. */ +typedef struct { + int64_t chart; + int64_t img; + bool literal; + char *keys; /* JSON array (cbm_alloc); NULL: none yet */ +} k8s_cimg_t; + +/* JSON array `a` (owned, may be NULL) with the strings of JSON array `b` + * appended once; the result is owned. */ +static char *k8s_keys_union(char *a, const char *b) { + yyjson_doc *db = b ? yyjson_read(b, strlen(b), 0) : NULL; + yyjson_val *bv = db ? yyjson_doc_get_root(db) : NULL; + size_t idx, max; + yyjson_val *k; + yyjson_arr_foreach(bv, idx, max, k) { + const char *s = yyjson_get_str(k); + if (!s) { + continue; + } + char esc[CBM_SZ_1K]; + char quoted[CBM_SZ_1K + PAIR_LEN + PAIR_LEN]; + cbm_json_escape(esc, (int)sizeof(esc), s); + snprintf(quoted, sizeof(quoted), "\"%s\"", esc); + if (a && strstr(a, quoted)) { + continue; + } + size_t al = a ? strlen(a) : 0; + char *g = cbm_alloc(CBM_MEM_CLASS_OTHER, al + strlen(quoted) + 3); + if (!g) { + break; + } + if (a) { + snprintf(g, al + strlen(quoted) + 3, "%.*s,%s]", (int)(al - SKIP_ONE), a, quoted); + } else { + snprintf(g, strlen(quoted) + 3, "[%s]", quoted); + } + cbm_free(CBM_MEM_CLASS_OTHER, a); + a = g; + } + if (db) { + yyjson_doc_free(db); + } + return a; +} + static void k8s_link_chart_images(cbm_pipeline_ctx_t *ctx) { const cbm_gbuf_node_t **charts = NULL; int nc = 0; if (cbm_gbuf_find_by_label(ctx->gbuf, "Chart", &charts, &nc) != 0 || nc == 0) { return; } - int64_t *pairs = NULL; - int np = 0; + k8s_cimg_t *v = NULL; + int n = 0; int cap = 0; for (int c = 0; c < nc; c++) { const cbm_gbuf_edge_t **dep = NULL; @@ -3748,26 +4915,41 @@ static void k8s_link_chart_images(cbm_pipeline_ctx_t *ctx) { !strstr(up, "\"ref\":\"image\"")) { continue; } - if (np + PAIR_LEN > cap) { - int ncap = cap ? cap * PAIR_LEN : CBM_SZ_64; - int64_t *grown = - cbm_realloc(CBM_MEM_CLASS_OTHER, pairs, (size_t)ncap * sizeof(*pairs)); - if (!grown) { + int at = -SKIP_ONE; + for (int i = n - SKIP_ONE; i >= 0 && at < 0; i--) { + at = v[i].chart == charts[c]->id && v[i].img == img->id ? i : at; + } + if (at < 0) { + if (!k8s_grow((void **)&v, &cap, n, sizeof(*v))) { break; } - pairs = grown; - cap = ncap; + v[n] = (k8s_cimg_t){charts[c]->id, img->id, false, NULL}; + at = n++; + } + const char *dflt = strstr(up, "\"default_of\":"); + if (!dflt) { + v[at].literal = true; + } else { + char keys[CBM_SZ_4K]; + const char *arr = dflt + strlen("\"default_of\":"); + const char *end = strchr(arr, ']'); + if (end && (size_t)(end - arr) + PAIR_LEN < sizeof(keys)) { + snprintf(keys, sizeof(keys), "%.*s", (int)(end - arr + SKIP_ONE), arr); + v[at].keys = k8s_keys_union(v[at].keys, keys); + } } - pairs[np++] = charts[c]->id; - pairs[np++] = img->id; } } } - for (int i = 0; i + SKIP_ONE < np; i += PAIR_LEN) { - cbm_gbuf_insert_edge(ctx->gbuf, pairs[i], pairs[i + SKIP_ONE], "DEPLOYS", - "{\"via\":\"chart\",\"ref\":\"image\"}"); + for (int i = 0; i < n; i++) { + char props[CBM_SZ_4K]; + bool marked = !v[i].literal && v[i].keys; + snprintf(props, sizeof(props), "{\"via\":\"chart\",\"ref\":\"image\"%s%s}", + marked ? ",\"default_of\":" : "", marked ? v[i].keys : ""); + cbm_gbuf_insert_edge(ctx->gbuf, v[i].chart, v[i].img, "DEPLOYS", props); + cbm_free(CBM_MEM_CLASS_OTHER, v[i].keys); } - cbm_free(CBM_MEM_CLASS_OTHER, pairs); + cbm_free(CBM_MEM_CLASS_OTHER, v); } /* A HelmRelease whose chartRef names OCIRepository `oci` (its USAGE edge @@ -3830,7 +5012,7 @@ static void k8s_prop_add(char *out, size_t out_sz, int *w, const char *key, cons * namespace, values, line} from the kustomization (its Kustomization * Resource, else its File -- the unit a Flux / Argo deploy reaches) to the * external Chart of that repository and name. */ -static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { +static void k8s_link_kcharts(k8s_dep_link_t *l, k8s_bundles_t *b, const k8s_facts_t *f) { for (int i = 0; i < f->kchart_n; i++) { const k8s_kchart_t *c = &f->kcharts[i]; if (!c->chart || strchr(c->chart, '/') || strchr(c->chart, '$')) { @@ -3838,8 +5020,18 @@ static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { } int64_t src = k8s_first_id(l->kusts, c->kust); src = src > 0 ? src : k8s_first_id(l->files, c->kust); + /* No repo: the chart in the kustomization's chartHome (charts/). */ + int64_t local = 0; + if (!c->url || !c->url[0]) { + char kdir[CBM_SZ_1K]; + char cdir[CBM_SZ_1K]; + k8s_dirname(c->kust, kdir, sizeof(kdir)); + snprintf(cdir, sizeof(cdir), "%s%scharts/%s", kdir, kdir[0] ? "/" : "", c->chart); + local = b->charts && cbm_ht_get(b->charts, cdir) ? k8s_chart_node(l, cdir) : 0; + } char props[CBM_SZ_4K]; - int w = snprintf(props, sizeof(props), "{\"via\":\"helmcharts\",\"external\":true"); + int w = snprintf(props, sizeof(props), "{\"via\":\"helmcharts\"%s", + local > 0 ? "" : ",\"external\":true"); k8s_prop_add(props, sizeof(props), &w, "chart", c->chart); k8s_prop_add(props, sizeof(props), &w, "url", c->url); k8s_prop_add(props, sizeof(props), &w, "version", c->version); @@ -3850,7 +5042,8 @@ static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { continue; } snprintf(props + w, sizeof(props) - (size_t)w, ",\"line\":%d}", c->line); - k8s_deploys_edge(l, src, k8s_external_chart(l, c->chart, c->url, c->kust), props); + k8s_deploys_edge( + l, src, local > 0 ? local : k8s_external_chart(l, c->chart, c->url, c->kust), props); } } @@ -4527,7 +5720,7 @@ static void k8s_link_deploys(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundl k8s_link_chart_refs(&l, &f->srcs[s]); } if (l.files && l.kusts) { - k8s_link_kcharts(&l, f); + k8s_link_kcharts(&l, b, f); } cbm_free(CBM_MEM_CLASS_OTHER, (void *)members.items); cbm_free(CBM_MEM_CLASS_OTHER, (void *)units.items); @@ -4603,6 +5796,14 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_chart_dir_add(&bundles, rel); helm_count++; break; + case K8S_WERF: { + char chart_dir[CBM_SZ_1K]; + if (handle_werf_chart(ctx, rel, p->source, chart_dir, sizeof(chart_dir))) { + k8s_chart_dir_register(&bundles, chart_dir); + helm_count++; + } + break; + } case K8S_KUSTOMIZE: k8s_kusts_push(&kusts, rel, p->source); k8s_kustomize_keep(&facts, rel, p->res); @@ -4647,6 +5848,9 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_crds(ctx, &bundles, &facts); k8s_link_deploys(ctx, &kusts, &bundles, &facts); k8s_link_images(ctx, &facts); + k8s_helm_t helm; + k8s_helm_init(&helm); + k8s_helm_collect(ctx, &facts, &helm); k8s_link_envs(ctx, &facts); k8s_link_patches(ctx, &kusts, &facts); k8s_link_renames(ctx, &kusts); @@ -4657,11 +5861,17 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_local_deps(ctx, &local_deps); k8s_link_includes(ctx, &bundles, files, file_count); k8s_link_chart_contents(ctx, &bundles); - k8s_bundles_free(&bundles); + /* The images chart templates render with their values (helm_template.c). */ + k8s_helm_link_defaults(ctx, &bundles, &helm); k8s_kusts_free(&kusts); /* Shell and Make deploy commands, against the nodes linked above. */ cbm_pipeline_pass_shell(ctx, files, file_count); cbm_pipeline_pass_terraform(ctx, files, file_count); + /* The images releases render (their values over the charts' defaults), + * before the image hubs move references. */ + k8s_helm_link_releases(ctx, &bundles, &helm); + k8s_helm_free(&helm); + k8s_bundles_free(&bundles); cbm_pipeline_pass_ansible(ctx, files, file_count); cbm_pipeline_pass_docker(ctx, files, file_count); /* The Images a chart's objects use, once the image hubs moved them. */ diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index 8e6f82ce9..e3a1e768a 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -376,10 +376,12 @@ typedef struct { const char *ci; /* static: the CI running the command; NULL: none */ const char *job; /* pass-arena: its job; NULL: none */ bool chart; - bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ - bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ - bool build; /* the image `ext` is built from the Dockerfile `path` */ - bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */ + bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ + bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ + bool build; /* the image `ext` is built from the Dockerfile `path` */ + bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */ + const char *build_args; /* pass-arena, build: --build-arg as a JSON object; NULL: none */ + const char *default_of; /* pass-arena, build: JSON array of the ${X:-d} defaults naming `ext` */ } sh_hit_t; typedef struct { @@ -413,9 +415,10 @@ typedef struct { const char *s; /* scalar value; NULL: unknown */ const char **items; /* array items (an item NULL: unknown); NULL: no array */ int n; + const char *dflt_of; /* JSON array: the variables whose ${X:-d} default made it */ } sh_val_t; -static const sh_val_t SH_UNKNOWN_VAL = {NULL, NULL, 0}; +static const sh_val_t SH_UNKNOWN_VAL = {NULL, NULL, 0, NULL}; typedef enum { SH_FREE, SH_AT, SH_LOST } sh_cwd_mode_t; @@ -464,8 +467,38 @@ typedef struct { sh_cwd_t *dirs; /* pushd stack */ int ndirs; int cap_dirs; + /* While set, an evaluation adds to dflt_acc (a JSON array) the variables + * whose ${X:-d} default supplied its value. */ + bool dflt_track; + const char *dflt_acc; } sh_t; +/* JSON array `list` (NULL: empty) with identifier `name` appended once. */ +static const char *sh_names_add(CBMArena *a, const char *list, const char *name, size_t nl) { + if (!list) { + return cbm_arena_sprintf(a, "[\"%.*s\"]", (int)nl, name); + } + char quoted[CBM_SZ_256]; + snprintf(quoted, sizeof(quoted), "\"%.*s\"", (int)nl, name); + if (strstr(list, quoted)) { + return list; + } + return cbm_arena_sprintf(a, "%.*s,%s]", (int)(strlen(list) - SKIP_ONE), list, quoted); +} + +/* JSON array `a` with the names of JSON array `b` appended once. */ +static const char *sh_names_union(CBMArena *ar, const char *a, const char *b) { + for (const char *p = b ? strchr(b, '"') : NULL; p;) { + const char *e = strchr(p + SKIP_ONE, '"'); + if (!e) { + break; + } + a = sh_names_add(ar, a, p + SKIP_ONE, (size_t)(e - p - SKIP_ONE)); + p = strchr(e + SKIP_ONE, '"'); + } + return a; +} + static void sh_set(sh_t *s, const char *name, const sh_val_t *v, bool local) { if (!sh_grow((void **)&s->undo, &s->cap_undo, s->nundo, sizeof(*s->undo))) { return; @@ -622,14 +655,14 @@ static const sh_val_t *sh_lookup(sh_t *s, const char *name, bool *found) { } sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); if (v) { - *v = (sh_val_t){s->script, NULL, 0}; + *v = (sh_val_t){s->script, NULL, 0, NULL}; } return v ? v : &SH_UNKNOWN_VAL; } if (strcmp(name, "PWD") == 0) { sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); if (v) { - *v = (sh_val_t){sh_cwd_value(s), NULL, 0}; + *v = (sh_val_t){sh_cwd_value(s), NULL, 0, NULL}; } return v ? v : &SH_UNKNOWN_VAL; } @@ -745,20 +778,29 @@ static const char *sh_eval_expansion(sh_t *s, TSNode n) { val = v->n > 0 ? v->items[0] : ""; } if (!op) { + if (found && s->dflt_track) { + s->dflt_acc = sh_names_union(s->a, s->dflt_acc, v->dflt_of); + } return found ? val : NULL; } bool colon = op[0] == ':'; bool use_default = !found || (colon && val && !val[0]); if (!use_default) { + if (s->dflt_track) { + s->dflt_acc = sh_names_union(s->a, s->dflt_acc, v->dflt_of); + } return val; } sh_str_t b = {0}; sh_concat(s, &b, n, op_end, close, true); const char *d = sh_str_done(&b); + if (s->dflt_track) { + s->dflt_acc = sh_names_add(s->a, s->dflt_acc, nm, strlen(nm)); + } if (d && strchr(op, '=')) { sh_val_t *nv = cbm_arena_alloc(s->a, sizeof(*nv)); if (nv) { - *nv = (sh_val_t){d, NULL, 0}; + *nv = (sh_val_t){d, NULL, 0, sh_names_add(s->a, NULL, nm, strlen(nm))}; sh_set(s, nm, nv, false); } } @@ -943,6 +985,9 @@ static const char *sh_eval(sh_t *s, TSNode n) { } bool found = false; const sh_val_t *val = sh_lookup(s, nm, &found); + if (found && s->dflt_track) { + s->dflt_acc = sh_names_union(s->a, s->dflt_acc, val->dflt_of); + } return found ? val->s : NULL; } if (strcmp(t, "expansion") == 0) { @@ -1319,6 +1364,8 @@ static void sh_hit_push(sh_t *s, TSNode cmd, sh_hit_t x) { x.line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base; x.ext = x.ext ? cbm_arena_strdup(&h->arena, x.ext) : NULL; x.path = x.path ? cbm_arena_strdup(&h->arena, x.path) : NULL; + x.build_args = x.build_args ? cbm_arena_strdup(&h->arena, x.build_args) : NULL; + x.default_of = x.default_of ? cbm_arena_strdup(&h->arena, x.default_of) : NULL; x.ci = s->ci; x.job = s->job ? cbm_arena_strdup(&h->arena, s->job) : NULL; h->items[h->count++] = x; @@ -2268,7 +2315,11 @@ static void sh_assign(sh_t *s, TSNode n, bool local) { if (!nm || !v) { return; } - *v = (sh_val_t){"", NULL, 0}; + *v = (sh_val_t){"", NULL, 0, NULL}; + bool track = s->dflt_track; + const char *acc = s->dflt_acc; + s->dflt_track = true; + s->dflt_acc = NULL; if (sh_type(value, "array")) { uint32_t cnt = ts_node_named_child_count(value); const char **items = cbm_arena_alloc(s->a, (cnt ? cnt : SKIP_ONE) * sizeof(*items)); @@ -2282,10 +2333,13 @@ static void sh_assign(sh_t *s, TSNode n, bool local) { items[k++] = sh_eval_word(s, c); } } - *v = (sh_val_t){k > 0 ? items[0] : "", items, k}; + *v = (sh_val_t){k > 0 ? items[0] : "", items, k, NULL}; } else if (!ts_node_is_null(value)) { v->s = sh_eval(s, value); } + v->dflt_of = s->dflt_acc; + s->dflt_track = track; + s->dflt_acc = acc; bool append = false; for (uint32_t i = 0; i < ts_node_child_count(n); i++) { TSNode c = ts_node_child(n, i); @@ -2434,6 +2488,54 @@ static const char *sh_image_repo(sh_t *s, TSNode cmd, const sh_argv_t *a, int i) return sh_str_done(&out); } +/* JSON object `json` (NULL: none) with build argument `kv` ("K=V") added; a + * `K` alone (the environment's value) or an unknown word adds nothing. */ +static const char *sh_build_arg(sh_t *s, const char *json, const char *kv) { + const char *eq = kv ? strchr(kv, '=') : NULL; + if (!eq || eq == kv) { + return json; + } + for (const char *p = kv; p < eq; p++) { + if (!isalnum((unsigned char)*p) && *p != '_') { + return json; + } + } + char ev[CBM_SZ_1K]; + cbm_json_escape(ev, (int)sizeof(ev), eq + SKIP_ONE); + const char *pair = cbm_arena_sprintf(s->a, "\"%.*s\":\"%s\"", (int)(eq - kv), kv, ev); + if (!pair) { + return json; + } + return json ? cbm_arena_sprintf(s->a, "%.*s,%s}", (int)(strlen(json) - SKIP_ONE), json, pair) + : cbm_arena_sprintf(s->a, "{%s}", pair); +} + +/* The variables whose ${X:-d} default supplied argv word `i` (a JSON array, + * NULL: none): the word evaluated once more with the defaults recorded. */ +static const char *sh_word_defaults(sh_t *s, TSNode cmd, const sh_argv_t *a, int i) { + uint32_t b = a->span[(size_t)i * PAIR_LEN]; + uint32_t e = a->span[(size_t)i * PAIR_LEN + SKIP_ONE]; + if (!memchr(s->src + b, '$', e - b)) { + return NULL; + } + TSNode w = ts_node_descendant_for_byte_range(cmd, b, e); + while (!ts_node_is_null(w) && (ts_node_start_byte(w) != b || ts_node_end_byte(w) != e)) { + w = ts_node_parent(w); + } + if (ts_node_is_null(w)) { + return NULL; + } + bool track = s->dflt_track; + const char *acc = s->dflt_acc; + s->dflt_track = true; + s->dflt_acc = NULL; + (void)sh_eval_word(s, w); + const char *out = s->dflt_acc; + s->dflt_track = track; + s->dflt_acc = acc; + return out; +} + /* docker / podman / nerdctl [buildx | image | builder] build, buildah bud: * each tagged image is built from the Dockerfile -f names (against the cwd), * else /Dockerfile -- one file of the repository. */ @@ -2467,17 +2569,20 @@ static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { cbm_ht_set(s->hits->declarers, s->file, (void *)s->file); } int tags[SH_BUILD_TAGS]; + int tag_words[SH_BUILD_TAGS]; const char *tag_vals[SH_BUILD_TAGS]; int nt = 0; const char *dfile = NULL; bool dfile_known = true; const char *ctx = NULL; bool ctx_seen = false; + const char *args = NULL; for (; i < a->n; i++) { const char *w = a->v[i]; if (w && (sh_eq(w, "-t") || sh_eq(w, "--tag")) && i + SKIP_ONE < a->n) { if (nt < SH_BUILD_TAGS) { tags[nt] = i + SKIP_ONE; + tag_words[nt] = i + SKIP_ONE; tag_vals[nt++] = a->v[i + SKIP_ONE]; } i++; @@ -2485,9 +2590,15 @@ static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { dfile = a->v[i + SKIP_ONE]; dfile_known = dfile != NULL; i++; + } else if (w && sh_eq(w, "--build-arg") && i + SKIP_ONE < a->n) { + args = sh_build_arg(s, args, a->v[i + SKIP_ONE]); + i++; + } else if (w && strncmp(w, "--build-arg=", strlen("--build-arg=")) == 0) { + args = sh_build_arg(s, args, w + strlen("--build-arg=")); } else if (w && strncmp(w, "--tag=", strlen("--tag=")) == 0) { if (nt < SH_BUILD_TAGS && w[strlen("--tag=")]) { tags[nt] = -SKIP_ONE; + tag_words[nt] = i; tag_vals[nt++] = w + strlen("--tag="); } } else if (w && strncmp(w, "--file=", strlen("--file=")) == 0) { @@ -2513,7 +2624,12 @@ static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { } if (img && img[0]) { sh_hit_push(s, cmd, - (sh_hit_t){.via = "docker-build", .ext = img, .path = hit, .build = true}); + (sh_hit_t){.via = "docker-build", + .ext = img, + .path = hit, + .build = true, + .build_args = args, + .default_of = sh_word_defaults(s, cmd, a, tag_words[k])}); } } } @@ -3305,8 +3421,8 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l sh_val_t *root = cbm_arena_alloc(&a, sizeof(*root)); sh_val_t *cur = cbm_arena_alloc(&a, sizeof(*cur)); if (root && cur) { - *root = (sh_val_t){SH_ROOT, NULL, 0}; - *cur = (sh_val_t){sh_anchor(&a, s.sdir), NULL, 0}; + *root = (sh_val_t){SH_ROOT, NULL, 0, NULL}; + *cur = (sh_val_t){sh_anchor(&a, s.sdir), NULL, 0, NULL}; cbm_ht_set(s.env, "__cbm_root", root); if (m.makefile) { cbm_ht_set(s.env, "__cbm_curdir", cur); @@ -3653,8 +3769,10 @@ static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTabl sh_ci_props(x, ci, sizeof(ci)); snprintf(props, sizeof(props), "{\"via\":\"build\",\"tool\":\"docker-build\",\"file\":\"%s\",\"image\":\"%s\"%s," - "\"line\":%d}", - esc_file, esc_img, ci, x->line); + "\"line\":%d%s%s%s%s}", + esc_file, esc_img, ci, x->line, x->default_of ? ",\"default_of\":" : "", + x->default_of ? x->default_of : "", x->build_args ? ",\"build_args\":" : "", + x->build_args ? x->build_args : ""); cbm_gbuf_insert_edge(ctx->gbuf, img, mod, "DEPENDS_ON", props); return SKIP_ONE; } @@ -3852,7 +3970,7 @@ typedef struct { static void sh_ci_var(sh_t *s, const char *name, const char *value) { sh_val_t *v = value ? cbm_arena_alloc(s->a, sizeof(*v)) : NULL; if (v) { - *v = (sh_val_t){value, NULL, 0}; + *v = (sh_val_t){value, NULL, 0, NULL}; } sh_set(s, name, v ? v : &SH_UNKNOWN_VAL, false); } diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c index 6328311c1..443bc4961 100644 --- a/src/pipeline/pass_terraform.c +++ b/src/pipeline/pass_terraform.c @@ -22,7 +22,9 @@ #include "foundation/log.h" #include "foundation/mem_core.h" #include "foundation/str_util.h" +#include "foundation/json_writer.h" /* cbm_json_writer_allocator */ #include "graph_buffer/graph_buffer.h" +#include "pipeline/helm_template.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "cbm.h" @@ -249,6 +251,402 @@ static bool tf_plain_name(const char *s) { } /* One `resource "helm_release" "" { ... }` block. */ +/* ── helm_release values (for pass_k8s.c's release images) ────────── */ + +/* The expression inside wrapper nodes. */ +static TSNode tf_unwrap(TSNode n) { + while (tf_type(n, "expression") || tf_type(n, "literal_value") || + tf_type(n, "collection_value") || tf_type(n, "template_expr")) { + if (ts_node_named_child_count(n) != SKIP_ONE) { + break; + } + n = ts_node_named_child(n, 0); + } + return n; +} + +/* A heredoc's text (a `<<-` one dedented); NULL when it interpolates. */ +static char *tf_heredoc(const tf_t *t, TSNode n) { + TSNode open = {0}; + TSNode close = {0}; + bool dedent = false; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "heredoc_start")) { + uint32_t b = ts_node_start_byte(c); + dedent = ts_node_end_byte(c) - b >= 3 && t->src[b + PAIR_LEN] == '-'; + } else if (tf_type(c, "heredoc_identifier")) { + if (ts_node_is_null(open)) { + open = c; + } else { + close = c; + } + } else if (!tf_type(c, "template_literal")) { + return NULL; /* an interpolation or a directive: no literal */ + } + } + if (ts_node_is_null(open) || ts_node_is_null(close)) { + return NULL; + } + const char *b = t->src + ts_node_end_byte(open); + const char *e = t->src + ts_node_start_byte(close); + const char *nl = memchr(b, '\n', (size_t)(e - b)); + b = nl ? nl + SKIP_ONE : e; + while (e > b && e[-1] != '\n') { + e--; /* the closing line's indentation */ + } + size_t indent = (size_t)-1; + for (const char *p = b; dedent && p < e;) { + const char *le = memchr(p, '\n', (size_t)(e - p)); + le = le ? le : e; + size_t sp = 0; + while (p + sp < le && (p[sp] == ' ' || p[sp] == '\t')) { + sp++; + } + if (p + sp < le && sp < indent) { + indent = sp; + } + p = le + (le < e ? SKIP_ONE : 0); + } + if (!dedent || indent == (size_t)-1) { + indent = 0; + } + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)(e - b) + SKIP_ONE); + size_t k = 0; + for (const char *p = b; out && p < e;) { + const char *le = memchr(p, '\n', (size_t)(e - p)); + le = le ? le + SKIP_ONE : e; + size_t skip = 0; + while (skip < indent && p + skip < le && (p[skip] == ' ' || p[skip] == '\t')) { + skip++; + } + memcpy(out + k, p + skip, (size_t)(le - p) - skip); + k += (size_t)(le - p) - skip; + p = le; + } + if (out) { + out[k] = '\0'; + } + return out; +} + +/* An HCL literal as JSON: a string without interpolation, a number, a bool, + * null, a tuple or object of those; NULL: anything else. */ +static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { + n = tf_unwrap(n); + if (tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { + char buf[CBM_SZ_4K]; + size_t len = 0; + buf[0] = '\0'; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "quoted_template_start") || tf_type(c, "quoted_template_end")) { + continue; + } + if (!tf_type(c, "template_literal")) { + return NULL; + } + uint32_t b = ts_node_start_byte(c); + if (!tf_add(buf, sizeof(buf), &len, t->src + b, ts_node_end_byte(c) - b)) { + return NULL; + } + } + size_t k = 0; /* \" \\ \n \t */ + for (size_t i = 0; i < len; i++) { + if (buf[i] == '\\' && i + SKIP_ONE < len) { + char c = buf[++i]; + buf[k++] = c == 'n' ? '\n' : c == 't' ? '\t' : c; + } else { + buf[k++] = buf[i]; + } + } + return yyjson_mut_strncpy(doc, buf, k); + } + if (tf_type(n, "heredoc_template")) { + char *s = tf_heredoc(t, n); + yyjson_mut_val *v = s ? yyjson_mut_strcpy(doc, s) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, s); + return v; + } + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (tf_type(n, "numeric_lit")) { + char buf[CBM_SZ_64]; + snprintf(buf, sizeof(buf), "%.*s", (int)(e - b), t->src + b); + return yyjson_mut_real(doc, strtod(buf, NULL)); + } + if (tf_type(n, "bool_lit")) { + return yyjson_mut_bool(doc, e - b == strlen("true") && strncmp(t->src + b, "true", 4) == 0); + } + if (tf_type(n, "null_lit")) { + return yyjson_mut_null(doc); + } + if (tf_type(n, "tuple")) { + yyjson_mut_val *arr = yyjson_mut_arr(doc); + for (uint32_t i = 0; arr && i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "tuple_start") || tf_type(c, "tuple_end")) { + continue; + } + yyjson_mut_val *v = tf_json(t, doc, c); + if (!v) { + return NULL; + } + yyjson_mut_arr_append(arr, v); + } + return arr; + } + if (tf_type(n, "object")) { + yyjson_mut_val *obj = yyjson_mut_obj(doc); + for (uint32_t i = 0; obj && i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (!tf_type(c, "object_elem")) { + continue; + } + TSNode kn = tf_unwrap(ts_node_child_by_field_name(c, "key", (uint32_t)strlen("key"))); + TSNode vn = ts_node_child_by_field_name(c, "val", (uint32_t)strlen("val")); + yyjson_mut_val *key = NULL; + if (tf_type(kn, "variable_expr") && ts_node_named_child_count(kn) == SKIP_ONE) { + TSNode id = ts_node_named_child(kn, 0); + uint32_t kb = ts_node_start_byte(id); + key = yyjson_mut_strncpy(doc, t->src + kb, ts_node_end_byte(id) - kb); + } else { + key = tf_json(t, doc, kn); + } + yyjson_mut_val *val = tf_json(t, doc, vn); + if (!key || !yyjson_mut_is_str(key) || !val) { + return NULL; + } + (void)yyjson_mut_obj_put(obj, key, val); + } + return obj; + } + return NULL; +} + +/* A repository file's text (cbm_alloc), NULL when unreadable. */ +static char *tf_read_repo(const tf_t *t, const char *path, size_t *len) { + char full[CBM_SZ_4K]; + snprintf(full, sizeof(full), "%s/%s", t->ctx->repo_path, path); + FILE *f = cbm_fopen(full, "rb"); + if (!f) { + return NULL; + } + char *buf = NULL; + size_t cap = 0; + char chunk[CBM_SZ_4K]; + size_t r; + *len = 0; + while ((r = fread(chunk, SKIP_ONE, sizeof(chunk), f)) > 0) { + if (*len + r + SKIP_ONE > cap) { + cap = (*len + r + SKIP_ONE) * PAIR_LEN; + char *g = cbm_realloc(CBM_MEM_CLASS_OTHER, buf, cap); + if (!g) { + cbm_free(CBM_MEM_CLASS_OTHER, buf); + (void)fclose(f); + return NULL; + } + buf = g; + } + memcpy(buf + *len, chunk, r); + *len += r; + } + (void)fclose(f); + if (buf) { + buf[*len] = '\0'; + } + return buf; +} + +/* A `values = [...]` item as values: a YAML string / heredoc, file() + * of a repository file, yamlencode / jsonencode of a literal; NULL: none of + * those. */ +static yyjson_mut_val *tf_values_item(const tf_t *t, yyjson_mut_doc *doc, TSNode item, CBMArena *a, + const char ***files, int *nfiles) { + TSNode n = tf_unwrap(item); + if (tf_type(n, "function_call")) { + TSNode fn = ts_node_named_child(n, 0); + TSNode args = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + if (tf_type(ts_node_named_child(n, i), "function_arguments")) { + args = ts_node_named_child(n, i); + } + } + if (ts_node_is_null(fn) || ts_node_is_null(args) || + ts_node_named_child_count(args) != SKIP_ONE) { + return NULL; + } + uint32_t fb = ts_node_start_byte(fn); + size_t fl = ts_node_end_byte(fn) - fb; + TSNode arg = ts_node_named_child(args, 0); + if ((fl == strlen("yamlencode") && strncmp(t->src + fb, "yamlencode", fl) == 0) || + (fl == strlen("jsonencode") && strncmp(t->src + fb, "jsonencode", fl) == 0)) { + return tf_json(t, doc, arg); + } + if (fl == strlen("file") && strncmp(t->src + fb, "file", fl) == 0) { + char v[CBM_SZ_1K]; + char path[CBM_SZ_1K]; + if (!t->ctx->repo_path || !tf_string(t->src, t->mdir, arg, v, sizeof(v)) || + !tf_repo_path(t->mdir, v, path, sizeof(path))) { + return NULL; + } + const char **grown = cbm_arena_alloc(a, (size_t)(*nfiles + SKIP_ONE) * sizeof(*grown)); + if (grown) { + if (*nfiles) { + memcpy(grown, *files, (size_t)*nfiles * sizeof(*grown)); + } + grown[(*nfiles)++] = cbm_arena_strdup(a, path); + *files = grown; + } + size_t len = 0; + char *buf = tf_read_repo(t, path, &len); + yyjson_mut_val *out = buf ? cbm_helm_values_parse(doc, buf, len) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return out; + } + return NULL; + } + yyjson_mut_val *s = tf_json(t, doc, n); + return yyjson_mut_is_str(s) + ? cbm_helm_values_parse(doc, yyjson_mut_get_str(s), yyjson_mut_get_len(s)) + : NULL; +} + +/* A `set` entry (name / value / type) applied to root; false when it is no + * literal. */ +static bool tf_set_entry(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_val *entry) { + yyjson_mut_val *nm = yyjson_mut_obj_get(entry, "name"); + yyjson_mut_val *val = yyjson_mut_obj_get(entry, "value"); + yyjson_mut_val *ty = yyjson_mut_obj_get(entry, "type"); + if (!yyjson_mut_is_str(nm) || !val) { + return false; + } + char vbuf[CBM_SZ_64]; + const char *v = NULL; + if (yyjson_mut_is_str(val)) { + v = yyjson_mut_get_str(val); + } else if (yyjson_mut_is_real(val)) { + snprintf(vbuf, sizeof(vbuf), "%.17g", yyjson_mut_get_real(val)); + v = vbuf; + } else if (yyjson_mut_is_bool(val)) { + v = yyjson_mut_get_bool(val) ? "true" : "false"; + } + bool as_string = yyjson_mut_is_str(ty) && strcmp(yyjson_mut_get_str(ty), "string") == 0; + return v && cbm_helm_values_set(doc, root, yyjson_mut_get_str(nm), v, as_string); +} + +/* A `set` block's attributes as an object; NULL when one is no literal. */ +static yyjson_mut_val *tf_set_block(const tf_t *t, yyjson_mut_doc *doc, TSNode block) { + yyjson_mut_val *entry = yyjson_mut_obj(doc); + for (uint32_t j = 0; entry && j < ts_node_named_child_count(block); j++) { + TSNode sb = ts_node_named_child(block, j); + if (!tf_type(sb, "body")) { + continue; + } + for (uint32_t k = 0; k < ts_node_named_child_count(sb); k++) { + TSNode at = ts_node_named_child(sb, k); + if (!tf_type(at, "attribute") || ts_node_named_child_count(at) < PAIR_LEN) { + continue; + } + TSNode key = ts_node_named_child(at, 0); + uint32_t b = ts_node_start_byte(key); + yyjson_mut_val *v = tf_json(t, doc, ts_node_named_child(at, SKIP_ONE)); + if (!v) { + return NULL; + } + yyjson_mut_obj_add(entry, + yyjson_mut_strncpy(doc, t->src + b, ts_node_end_byte(key) - b), v); + } + } + return entry; +} + +/* The values a helm_release body gives its chart -- `values` items + * coalesced in order, then each `set` -- as JSON (cbm_alloc); NULL when a + * part is no literal. set_sensitive is never read. */ +static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + bool ok = root != NULL; + for (int pass = 0; ok && pass < PAIR_LEN; pass++) { + for (uint32_t i = 0; ok && i < ts_node_named_child_count(body); i++) { + TSNode a = ts_node_named_child(body, i); + TSNode key = ts_node_named_child(a, 0); + uint32_t kb = ts_node_is_null(key) ? 0 : ts_node_start_byte(key); + size_t kl = ts_node_is_null(key) ? 0 : ts_node_end_byte(key) - kb; + bool is_set = kl == strlen("set") && strncmp(t->src + kb, "set", kl) == 0; + bool is_values = kl == strlen("values") && strncmp(t->src + kb, "values", kl) == 0; + /* set_list values are not read: the release's values are partial */ + if (kl == strlen("set_list") && strncmp(t->src + kb, "set_list", kl) == 0) { + ok = false; + break; + } + if (pass == 0 && is_values && tf_type(a, "attribute")) { + TSNode list = tf_unwrap(ts_node_named_child(a, SKIP_ONE)); + ok = tf_type(list, "tuple"); + for (uint32_t j = 0; ok && j < ts_node_named_child_count(list); j++) { + TSNode it = ts_node_named_child(list, j); + if (tf_type(it, "tuple_start") || tf_type(it, "tuple_end")) { + continue; + } + yyjson_mut_val *v = tf_values_item(t, doc, it, t->arena, files, nfiles); + ok = v && yyjson_mut_is_obj(v); + root = ok ? cbm_helm_values_coalesce(doc, root, v) : root; + } + } else if (pass == SKIP_ONE && is_set && tf_type(a, "block")) { + yyjson_mut_val *entry = tf_set_block(t, doc, a); + ok = entry && tf_set_entry(doc, root, entry); + } else if (pass == SKIP_ONE && is_set && tf_type(a, "attribute")) { + /* provider v3: set = [{ name, value, type }, ...] */ + yyjson_mut_val *list = tf_json(t, doc, ts_node_named_child(a, SKIP_ONE)); + ok = yyjson_mut_is_arr(list); + size_t idx, max; + yyjson_mut_val *entry; + yyjson_mut_arr_foreach(list, idx, max, entry) { + ok = ok && tf_set_entry(doc, root, entry); + } + } + } + } + char *out = NULL; + if (ok) { + size_t len = 0; + out = yyjson_mut_val_write_opts(root, 0, cbm_json_writer_allocator(), &len, NULL); + } + if (doc) { + yyjson_mut_doc_free(doc); + } + return out; +} + +static void tf_release_push(tf_t *t, int64_t src, int64_t chart, TSNode body) { + cbm_pipeline_ctx_t *ctx = t->ctx; + if (ctx->tf_release_n == ctx->tf_release_cap) { + int cap = ctx->tf_release_cap ? ctx->tf_release_cap * PAIR_LEN : CBM_SZ_16; + cbm_tf_release_t *g = + cbm_realloc(CBM_MEM_CLASS_OTHER, ctx->tf_releases, (size_t)cap * sizeof(*g)); + if (!g) { + return; + } + ctx->tf_releases = g; + ctx->tf_release_cap = cap; + } + const char **files = NULL; + int nfiles = 0; + ctx->tf_releases[ctx->tf_release_n++] = + (cbm_tf_release_t){src, chart, tf_release_values(t, body, &files, &nfiles)}; + /* the values files it reads, like a kustomization's valuesFile: an edit of + * one alone renders the release again (pipeline_incremental.c) */ + for (int i = 0; i < nfiles; i++) { + const char *mqn = cbm_fqn_module(t->arena, ctx->project_name, files[i]); + const cbm_gbuf_node_t *mod = mqn ? cbm_gbuf_find_by_qn(ctx->gbuf, mqn) : NULL; + if (mod && mod->id != src) { + cbm_gbuf_insert_edge(ctx->gbuf, src, mod->id, "IMPORTS", "{\"local_name\":\"values\"}"); + t->edges++; + } + } +} + static void tf_helm_release(tf_t *t, TSNode block, const char *name) { TSNode body = {0}; for (uint32_t i = 0; i < ts_node_named_child_count(block); i++) { @@ -320,6 +718,7 @@ static void tf_helm_release(tf_t *t, TSNode block, const char *name) { external ? ",\"external\":true" : "", (int)ts_node_start_point(block).row + SKIP_ONE); cbm_gbuf_insert_edge(t->ctx->gbuf, src, tgt, "DEPLOYS", props); t->edges++; + tf_release_push(t, src, tgt, body); } /* The text of a string_lit label: "helm_release" -> helm_release. */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 99df053e4..bb97c361d 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2456,8 +2456,9 @@ static bool closure_dir_has_compose(const char *repo_path, const char *rel, cons return found; } -/* The nearest directory at or above `rel`'s that holds a Chart.yaml, into out - * ("" for the repository root); false when none does. */ +/* The nearest directory at or above `rel`'s that holds a Chart.yaml, or is a + * werf `.helm` chart, into out ("" for the repository root); false when none + * is. */ static bool closure_chart_dir_of(const char *repo_path, const char *rel, char *out, size_t sz) { snprintf(out, sz, "%s", rel); char *cut = strrchr(out, '/'); @@ -2475,6 +2476,9 @@ static bool closure_chart_dir_of(const char *repo_path, const char *rel, char *o return true; } } + if (out[0] && cbm_werf_helm_dir(repo_path, out)) { + return true; + } if (!out[0]) { return false; } @@ -2482,6 +2486,33 @@ static bool closure_chart_dir_of(const char *repo_path, const char *rel, char *o } } +/* Is `rel` a file a release reads as values (an IMPORTS {local_name: + * valuesFile | values} edge reaches its node)? */ +static bool closure_release_values(cbm_store_t *store, const char *project, const char *rel) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { + return false; + } + bool hit = false; + for (int i = 0; i < count && !hit; i++) { + cbm_edge_t *edges = NULL; + int ne = 0; + if (cbm_store_find_edges_by_target_type(store, nodes[i].id, "IMPORTS", &edges, &ne) != + CBM_STORE_OK) { + continue; + } + for (int j = 0; j < ne && !hit; j++) { + const char *p = edges[j].properties_json; + hit = p && (strstr(p, "\"local_name\":\"valuesFile\"") || + strstr(p, "\"local_name\":\"values\"")); + } + cbm_store_free_edges(edges, ne); + } + cbm_store_free_nodes(nodes, count); + return hit; +} + /* Kubernetes inputs reach past their own file (pass_k8s.c): a kustomization * decides which manifests deploy together and in which namespace, a * Chart.yaml links other charts, and a manifest's Services select workloads @@ -2498,7 +2529,9 @@ static const char *closure_k8s_decline(cbm_store_t *store, const char *project, if (cbm_is_kustomize_file(base) || strcmp(base, "Kustomization") == 0) { return "k8s_bundle_input_changed"; } - if (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) { + /* A werf.yaml names the chart of the `.helm` dir beside it. */ + if (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0 || + strcmp(base, "werf.yaml") == 0 || strcmp(base, "werf.yml") == 0) { return "helm_chart_changed"; } /* A chart's values file CONFIGURES it and its templates/ are what it @@ -2518,6 +2551,12 @@ static const char *closure_k8s_decline(cbm_store_t *store, const char *project, if (strcmp(base, ".env") == 0 && closure_dir_has_compose(repo_path, rel, base)) { return "compose_env_changed"; } + /* A release's values file -- a kustomization's valuesFile, a Terraform + * helm_release's file(...) -- feeds the images the release renders, + * which a repair of the values file alone does not redraw. */ + if (closure_release_values(store, project, rel)) { + return "helm_release_values_changed"; + } cbm_node_t *nodes = NULL; int count = 0; if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index fbae8044f..553aa19d9 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -380,6 +380,16 @@ typedef struct { int reader_count; } cbm_envjoin_base_t; +/* A Terraform helm_release (pass_terraform.c): the chart it DEPLOYS and its + * values -- `values = [...]` coalesced in order, then `set` -- as JSON; NULL + * when a part is not a literal (an interpolation, a variable). pass_k8s.c + * renders the chart's images with them. */ +typedef struct cbm_tf_release { + int64_t src; /* the resource.helm_release node */ + int64_t chart; /* the Chart node */ + char *values; /* owned (cbm_free); NULL: unknown */ +} cbm_tf_release_t; + /* Shared context passed to each pass function. * Derived from cbm_pipeline_t fields during run. */ typedef struct { @@ -486,6 +496,12 @@ typedef struct { const struct cbm_doclink_scope *doc_link_base; int doc_link_base_count; bool doc_links_failed; + + /* The run's Terraform helm_releases (pass_terraform.c fills them, the K8s + * pass reads and frees them right after). */ + cbm_tf_release_t *tf_releases; + int tf_release_n; + int tf_release_cap; } cbm_pipeline_ctx_t; /* Origin-aware LSP returns only success/failure. Never retain partial output @@ -1060,6 +1076,9 @@ bool cbm_is_k8s_manifest(const char *name, const char *content); /* A Helm values file name: values.yaml / .yml, values-.yaml / .yml * (pass_k8s.c; beside Chart.yaml it CONFIGURES the chart). */ bool cbm_helm_values_name(const char *base); +/* Is `dir` (repository relative) a werf chart: a `.helm` directory without a + * Chart.yaml whose parent holds werf.yaml (pass_k8s.c)? */ +bool cbm_werf_helm_dir(const char *repo_path, const char *dir); /* Secret detection */ bool cbm_is_secret_binding(const char *key, const char *value); @@ -1259,6 +1278,7 @@ typedef struct { const char *via; /* "skaffold" / "compose" */ const char *image; /* the reference, variables expanded */ const char *dockerfile; /* repository path */ + const char *build_args; /* compose build.args as a JSON object; NULL: none */ int line; } cbm_build_site_t; typedef void (*cbm_build_site_fn)(const cbm_build_site_t *site, void *ud); @@ -1269,10 +1289,17 @@ typedef void (*cbm_build_site_fn)(const cbm_build_site_t *site, void *ud); * depends_on and links list, and the names of the environment variables its * environment sets (map keys / list items' names -- never a value), with * their 1-based lines and the service name each value addresses (a bare - * name, `name:port`, a URL's host; NULL: none -- cbm_env_address). */ + * name, `name:port`, a URL's host; NULL: none -- cbm_env_address). + * image_default_of: the variables whose inline default (${X:-d}) named the + * image; image_override_of: those whose default the project's .env replaced + * (JSON arrays, NULL: none), and default_image the image those defaults + * name. */ typedef struct { const char *name; const char *image; + const char *image_default_of; + const char *image_override_of; + const char *default_image; const char *dockerfile; const char **deps; int ndeps; @@ -1340,6 +1367,12 @@ bool cbm_repo_rel_join(const char *dir, const char *v, char *out, size_t sz); * for an official image; registry "unknown" when the reference names no * host -- none is invented), made when new; 0 for scratch or no image. */ int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref); +/* Does `ref` follow the image reference grammar (distribution/reference): + * [host[:port]/]path[:tag][@algo:hex], path components lowercase alphanumerics + * joined by `.`, `_`, `__` or dashes, a tag of word characters, `.` and `-` + * (128 at most)? A value rendered from a template is checked before it names + * an Image (pass_docker.c). */ +bool cbm_image_ref_valid(const char *ref); /* A chart repository URL as an identity (pass_k8s.c): `url` trimmed, its * scheme and host lower-cased, trailing `/` and `.git` dropped. False for no @@ -1730,7 +1763,8 @@ int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, con const char *label); /* The edges that keep a file-less shared node alive. */ #define SH_PROCESS_EDGES ((const char *const[]){"SPAWNS", NULL}) -#define DK_IMAGE_EDGES ((const char *const[]){"DEPENDS_ON", "USAGE", NULL}) +/* An Image a release DEPLOYS (pass_k8s.c release images) may have no other edge. */ +#define DK_IMAGE_EDGES ((const char *const[]){"DEPENDS_ON", "USAGE", "DEPLOYS", NULL}) /* Pre-seed proxy nodes with their REAL database ids and move the gbuf id * watermark above MAX(id); returns that max id, or -1 on failure. */ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index cebd5275b..f7405e959 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -4445,6 +4445,7 @@ static char *gpf_chart_edges(const GpfFile *files, int nfiles) { "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT CASE WHEN " "s.label='File' THEN s.file_path ELSE s.name END || ' -> ' || t.label || ':' || " "t.name || ' (' || e.type || coalesce(' ' || json_extract(e.properties,'$.ref'), '') " + "|| coalesce(' ' || json_extract(e.properties,'$.phase'), '') " "|| ')' AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " "t.id=e.target_id WHERE e.project=?1 AND ((s.label='Chart' AND e.type='DEPLOYS') OR " "(t.label='Chart' AND e.type='CONFIGURES')) ORDER BY x)"; @@ -4462,8 +4463,8 @@ static char *gpf_chart_edges(const GpfFile *files, int nfiles) { /* A chart deploys what its templates render: Chart -DEPLOYS {via: chart}-> * each object of its templates/ (a subchart's objects are the subchart's), - * and each Image those objects name literally ({ref: image}) -- a templated - * image names none, and no values default is read. Its values files + * and each Image those objects use ({ref: image}) -- a templated one rendered + * from values.yaml (`nginx`: host-less, kept as written). Its values files * (values.yaml, values-*.yaml beside Chart.yaml) CONFIGURE it; a values file * elsewhere (ci/) does not. */ TEST(probe_helm_chart_deploys_its_objects) { @@ -4492,6 +4493,7 @@ TEST(probe_helm_chart_deploys_its_objects) { "db -> Image:ghcr.io/acme/db (DEPLOYS image)\n" "db -> Resource:StatefulSet/db (DEPLOYS)\n" "web -> Image:ghcr.io/acme/sidecar (DEPLOYS image)\n" + "web -> Image:nginx (DEPLOYS image)\n" "web -> Resource:Deployment/web (DEPLOYS)\n" "web -> Resource:Service/web (DEPLOYS)"; if (!got || strcmp(got, want) != 0) { @@ -4503,6 +4505,267 @@ TEST(probe_helm_chart_deploys_its_objects) { PASS(); } +/* A chart also deploys the CRDs of its crds/ ({phase: crds}), which Helm + * installs before its templates. werf deploys the `.helm` dir beside its + * werf.yaml as a chart without Chart.yaml, named after the werf project + * (templated werf.yaml lines before it are skipped); a `.helm` with its own + * Chart.yaml stays that chart, and a werf.yaml without `.helm` makes none. */ +TEST(probe_helm_chart_crds_and_werf_chart) { + static const GpfFile files[] = { + {"charts/op/Chart.yaml", "apiVersion: v2\nname: op\nversion: 1.0.0\n"}, + {"charts/op/crds/widget.yaml", + "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n" + " name: widgets.acme.io\nspec:\n group: acme.io\n names:\n kind: Widget\n" + " plural: widgets\n scope: Namespaced\n"}, + {"charts/op/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: op\n"}, + {"site/werf.yaml", "{{- $_ := set . \"Env\" (.Env | default \"prod\") }}\n\n" + "project: site-web\nconfigVersion: 1\n---\nimage: web\nfrom: nginx\n"}, + {"site/.helm/values.yaml", "replicas: 1\n"}, + {"site/.helm/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: site\nspec:\n" + " replicas: {{ .Values.replicas }}\n"}, + {"plain/werf.yaml", "project: plain\nconfigVersion: 1\n"}, + {"own/werf.yaml", "project: own-werf\nconfigVersion: 1\n"}, + {"own/.helm/Chart.yaml", "apiVersion: v2\nname: own\nversion: 0.1.0\n"}, + {"own/.helm/templates/cm.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: own\n"}}; + char *got = gpf_chart_edges(files, 10); + static const char want[] = + "op -> Resource:CustomResourceDefinition/widgets.acme.io (DEPLOYS crds)\n" + "op -> Resource:Deployment/op (DEPLOYS)\n" + "own -> Resource:ConfigMap/own (DEPLOYS)\n" + "site-web -> Resource:Deployment/site (DEPLOYS)\n" + "site/.helm/values.yaml -> Chart:site-web (CONFIGURES)"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "chart edges got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("crds/ and werf chart edge set"); + } + free(got); + PASS(); +} + +/* Every edge into an Image (or out of one into an Image), one line each: + * `source -> Image:name TYPE image=.. [via=..] [ref=..] [default_of=..] + * [override_of=..] [inferred=..] [values_key=..]`, sorted. */ +static char *gpf_image_edges(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT s.label || ':' || s.name " + "|| ' -> Image:' || t.name || ' ' || e.type || ' image=' || " + "coalesce(json_extract(e.properties,'$.image'),'-') || " + "coalesce(' via=' || json_extract(e.properties,'$.via'), '') || " + "coalesce(' ref=' || json_extract(e.properties,'$.ref'), '') || " + "coalesce(' default_of=' || json_extract(e.properties,'$.default_of'), '') || " + "coalesce(' override_of=' || json_extract(e.properties,'$.override_of'), '') || " + "coalesce(' inferred=' || json_extract(e.properties,'$.inferred'), '') || " + "coalesce(' values_key=' || json_extract(e.properties,'$.values_key'), '') AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND t.label='Image' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +static bool gpf_expect_image_edges(const GpfFile *files, int n, const char *want) { + char *got = gpf_image_edges(files, n); + bool ok = got && strcmp(got, want) == 0; + if (!ok) { + fprintf(stderr, "image edges got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + } + free(got); + return ok; +} + +/* An image a default value names keeps its Image, its edge marked + * default_of with the ARGs / variables whose default supplied it (an ARG + * chain names each); a literal FROM is unmarked. A build's arguments (compose + * build.args, docker build --build-arg) make the Image it builds DEPEND_ON + * the base they choose {override_of}; compose's .env replacing an inline + * default is the effective edge's override_of, the default's image kept as + * {ref: image_default} -- one the service does not run. */ +TEST(probe_docker_default_images_marked_and_overrides) { + static const GpfFile files[] = { + {"src/web/Dockerfile", "ARG GO=1.22\nARG BASE=golang:${GO}\nFROM ${BASE} AS build\n" + "RUN go build -o /app .\nFROM alpine:3.20\n" + "COPY --from=build /app /app\n"}, + {"src/api/Dockerfile", "ARG PY=3.12\nFROM python:${PY}-slim\n"}, + {".env", "REG=ghcr.io/acme\nAPI_IMG=ghcr.io/acme/api-custom\n"}, + {"compose.yaml", "services:\n web:\n image: ${REG}/web:${TAG:-dev}\n build:\n" + " context: ./src/web\n args:\n GO: \"1.23\"\n" + " api:\n image: ${API_IMG:-ghcr.io/acme/api}\n build: ./src/api\n"}, + {"scripts/build.sh", "#!/bin/sh\nIMAGE=${IMAGE:-ghcr.io/acme/api-ci}\n" + "docker build --build-arg PY=3.13 -t \"$IMAGE:1\" src/api\n"}}; + static const char want[] = + "Image:ghcr.io/acme/api-ci -> Image:python DEPENDS_ON image=python:3.13-slim via=from " + "override_of=[\"PY\"]\n" + "Image:ghcr.io/acme/web -> Image:golang DEPENDS_ON image=golang:1.23 via=from " + "default_of=[\"BASE\"] override_of=[\"GO\"]\n" + "Module:src/api/Dockerfile -> Image:python DEPENDS_ON image=python:3.12-slim via=from " + "default_of=[\"PY\"]\n" + "Module:src/web/Dockerfile -> Image:alpine DEPENDS_ON image=alpine:3.20 via=from\n" + "Module:src/web/Dockerfile -> Image:golang DEPENDS_ON image=golang:1.22 via=from " + "default_of=[\"GO\",\"BASE\"]\n" + "Resource:Service/api -> Image:ghcr.io/acme/api USAGE image=ghcr.io/acme/api " + "ref=image_default default_of=[\"API_IMG\"]\n" + "Resource:Service/api -> Image:ghcr.io/acme/api-custom USAGE image=ghcr.io/acme/api-custom " + "ref=image override_of=[\"API_IMG\"]\n" + "Resource:Service/web -> Image:ghcr.io/acme/web USAGE image=ghcr.io/acme/web:dev " + "ref=image default_of=[\"TAG\"]"; + if (!gpf_expect_image_edges(files, 5, want)) { + FAIL("default-valued images and their in-repo overrides"); + } + PASS(); +} + +/* The chart a Helm template renders (helm_template.c): the container image + * from the chart family's defaults -- an include helper (bitnami + * common.images.image, a dict argument, .Chart.AppVersion), a `with` + * binding, the taken branch of an if / else, a subchart under its alias with + * the parent's values and global. The object USES it {default_of: the values + * keys read}; the chart DEPLOYS it. A value that does not render -- a + * .Release field, lookup, a missing define, a field of a missing map -- names + * no Image. */ +TEST(probe_helm_template_images_render_with_defaults) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", + "apiVersion: v2\nname: app\nversion: 1.2.3\nappVersion: \"2.0.1\"\ndependencies:\n" + " - name: db\n version: 0.1.0\n alias: database\n"}, + {"charts/app/values.yaml", + "image:\n registry: ghcr.io\n repository: acme/app\n tag: \"\"\n digest: \"\"\n" + "sidecar:\n image: { repository: docker.io/library/busybox, tag: 1.36 }\nglobal: {}\n" + "database:\n image:\n tag: \"17\"\n"}, + {"charts/app/templates/_helpers.tpl", + "{{- define \"common.images.image\" -}}\n" + "{{- $registryName := default .imageRoot.registry ((.global).imageRegistry) -}}\n" + "{{- $termination := .imageRoot.tag | toString -}}\n" + "{{- if not .imageRoot.tag }}\n {{- if .chart }}\n" + " {{- $termination = .chart.AppVersion | toString -}}\n {{- end -}}\n{{- end -}}\n" + "{{- if $registryName }}\n" + "{{- printf \"%s/%s:%s\" $registryName .imageRoot.repository $termination -}}\n" + "{{- else -}}\n{{- printf \"%s:%s\" .imageRoot.repository $termination -}}\n" + "{{- end -}}\n{{- end -}}\n"}, + {"charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n image: {{ include " + "\"common.images.image\" (dict \"imageRoot\" .Values.image \"global\" .Values.global " + "\"chart\" .Chart) }}\n - name: side\n {{- with .Values.sidecar.image }}\n" + " image: \"{{ .repository }}:{{ .tag }}\"\n {{- end }}\n" + " - name: rel\n image: \"ghcr.io/acme/{{ .Release.Name }}:1\"\n" + " - name: miss\n image: {{ include \"nope\" . }}\n" + " - name: deep\n image: \"{{ .Values.absent.repository }}\"\n"}, + {"charts/app/charts/db/Chart.yaml", + "apiVersion: v2\nname: db\nversion: 0.1.0\nappVersion: \"16.4\"\n"}, + {"charts/app/charts/db/values.yaml", + "image:\n repository: bitnami/postgresql\n tag: \"\"\n digest: \"\"\n"}, + {"charts/app/charts/db/templates/sts.yaml", + "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: db\nspec:\n template:\n" + " spec:\n containers:\n - name: db\n {{- if .Values.image.digest }}\n" + " image: \"{{ .Values.global.imageRegistry | default \"docker.io\" }}/" + "{{ .Values.image.repository }}@{{ .Values.image.digest }}\"\n {{- else }}\n" + " image: \"{{ .Values.global.imageRegistry | default \"docker.io\" }}/" + "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}\"\n" + " {{- end }}\n"}}; + static const char want[] = + "Chart:app -> Image:docker.io/library/busybox DEPLOYS image=- via=chart ref=image " + "default_of=[\"sidecar.image.repository\",\"sidecar.image.tag\"]\n" + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.registry\",\"global.imageRegistry\",\"image.tag\"," + "\"image.repository\"]\n" + "Chart:db -> Image:docker.io/bitnami/postgresql DEPLOYS image=- via=chart ref=image " + "default_of=[\"global.imageRegistry\",\"image.repository\",\"image.tag\"]\n" + "Resource:Deployment/app -> Image:docker.io/library/busybox USAGE " + "image=docker.io/library/busybox:1.36 ref=image " + "default_of=[\"sidecar.image.repository\",\"sidecar.image.tag\"]\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:2.0.1 " + "ref=image default_of=[\"image.registry\",\"global.imageRegistry\",\"image.tag\"," + "\"image.repository\"]\n" + "Resource:StatefulSet/db -> Image:docker.io/bitnami/postgresql USAGE " + "image=docker.io/bitnami/postgresql:17 ref=image " + "default_of=[\"global.imageRegistry\",\"image.repository\",\"image.tag\"]"; + if (!gpf_expect_image_edges(files, 7, want)) { + FAIL("Helm template images rendered with the chart family's defaults"); + } + PASS(); +} + +/* A release renders its chart with its values over the defaults: of a chart + * in the repository each image that differs is the release's own {via, + * override_of} -- a Flux HelmRelease's spec.values (a Git source), a + * kustomize helmCharts entry's valuesFile + valuesInline (chartHome charts/), + * a Terraform helm_release's values + set. Of an external chart the values + * name images by convention -- `repository` (+ tag / registry) naming a + * registry host -- {inferred: values}; a host-less repository names none. */ +TEST(probe_helm_release_images) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, + {"charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"}, + {"flux/app.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: app\nspec:\n" + " chart:\n spec:\n chart: ./charts/app\n sourceRef:\n" + " kind: GitRepository\n name: repo\n values:\n image:\n" + " tag: \"9.9\"\n---\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: sonarr\n" + "spec:\n chart:\n spec:\n chart: app-template\n sourceRef:\n" + " kind: HelmRepository\n name: bjw-s\n values:\n controllers:\n" + " main:\n containers:\n main:\n image:\n" + " repository: ghcr.io/home-operations/sonarr\n tag: 4.0.15\n" + " side:\n image:\n repository: busybox\n" + " tag: \"1.36\"\n"}, + {"kz/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nhelmCharts:\n" + " - name: app\n releaseName: kapp\n valuesFile: values-prod.yaml\n" + " valuesInline:\n image:\n tag: \"5.5\"\n"}, + {"kz/values-prod.yaml", "image:\n repository: quay.io/acme/app\n tag: \"4.4\"\n"}, + {"kz/charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, + {"kz/charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"kz/charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: kapp\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"}, + {"tf/main.tf", "resource \"helm_release\" \"app\" {\n name = \"app\"\n" + " chart = \"../charts/app\"\n set {\n name = \"image.tag\"\n" + " value = \"7.7\"\n }\n}\n"}}; + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.repository\",\"image.tag\"]\n" + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.repository\",\"image.tag\"]\n" + "Class:resource.helm_release.app -> Image:ghcr.io/acme/app DEPLOYS " + "image=ghcr.io/acme/app:7.7 via=terraform ref=image override_of=[\"image.tag\"]\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " + "ref=image default_of=[\"image.repository\",\"image.tag\"]\n" + "Resource:Deployment/kapp -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " + "ref=image default_of=[\"image.repository\",\"image.tag\"]\n" + "Resource:HelmRelease/app -> Image:ghcr.io/acme/app DEPLOYS image=ghcr.io/acme/app:9.9 " + "via=helmrelease ref=image override_of=[\"image.tag\"]\n" + "Resource:HelmRelease/sonarr -> Image:ghcr.io/home-operations/sonarr DEPLOYS " + "image=ghcr.io/home-operations/sonarr:4.0.15 via=helmrelease ref=image inferred=values " + "values_key=controllers.main.containers.main.image\n" + "Resource:Kustomization -> Image:quay.io/acme/app DEPLOYS image=quay.io/acme/app:5.5 " + "via=helmcharts ref=image override_of=[\"image.repository\",\"image.tag\"]"; + if (!gpf_expect_image_edges(files, 10, want)) { + FAIL("release images: local renders and the external values convention"); + } + PASS(); +} + /* A template's `include "name"` calls the define of that name in its chart * family -- the chart and its subcharts share one template namespace -- never * a same-named define of an unrelated chart; a define may also be written in @@ -6313,6 +6576,10 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_terraform_helm_release_deploys_its_chart); RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_chart_deploys_its_objects); + RUN_TEST(probe_helm_chart_crds_and_werf_chart); + RUN_TEST(probe_docker_default_images_marked_and_overrides); + RUN_TEST(probe_helm_template_images_render_with_defaults); + RUN_TEST(probe_helm_release_images); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index b52513ace..079df09da 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -15,6 +15,7 @@ #include "pipeline/lsp_surface.h" #include "pipeline/pass_lsp_cross.h" #include "pipeline/pipeline_internal.h" +#include "pipeline/helm_template.h" #include "pipeline/artifact.h" #include "store/store.h" #include "semantic/semantic.h" @@ -4205,6 +4206,135 @@ TEST(incremental_helm_values_edit_keeps_chart_edge) { PASS(); } +/* A werf `.helm` dir is a chart without Chart.yaml (pass_k8s.c): its values + * file CONFIGURES the werf Chart, an edge a closure repair of the values file + * alone does not hold, so the edit rebuilds, like one beside a Chart.yaml. */ +TEST(incremental_werf_values_edit_keeps_chart_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_werfchart_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "werf.yaml", "project: site\nconfigVersion: 1\n"); + write_temp_file(tmp, ".helm/values.yaml", "replicas: 1\n"); + write_temp_file(tmp, ".helm/templates/svc.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/werfchart.db", tmp); + static const char edge[] = + "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.type='CONFIGURES' AND s.file_path='.helm/values.yaml' AND " + "t.label='Chart' AND t.name='site'"; + int before = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, edge) : -1; + write_temp_file(tmp, ".helm/values.yaml", "replicas: 3\n"); + int after = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, edge) : -1; + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + th_rmtree(tmp); + ASSERT_EQ(before, 1); + ASSERT_EQ(after, 1); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + +/* A kustomization's helmCharts valuesFile (an IMPORTS {local_name: + * valuesFile} target) feeds the image the release renders: the values file + * edited alone rebuilds, and the release's image follows the edit. */ +TEST(incremental_release_values_edit_rebuilds) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_relvalues_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "kz/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "helmCharts:\n - name: app\n releaseName: kapp\n" + " valuesFile: values-prod.yaml\n"); + write_temp_file(tmp, "kz/values-prod.yaml", "image:\n tag: \"2.0\"\n"); + write_temp_file(tmp, "kz/charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"); + write_temp_file(tmp, "kz/charts/app/values.yaml", + "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"); + write_temp_file(tmp, "kz/charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n" + " template:\n spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/relvalues.db", tmp); + static const char v2[] = "SELECT COUNT(*) FROM edges WHERE type='DEPLOYS' AND " + "json_extract(properties,'$.image')='ghcr.io/acme/app:2.0'"; + static const char v3[] = "SELECT COUNT(*) FROM edges WHERE type='DEPLOYS' AND " + "json_extract(properties,'$.image')='ghcr.io/acme/app:3.0'"; + int before = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, v2) : -1; + write_temp_file(tmp, "kz/values-prod.yaml", "image:\n tag: \"3.0\"\n"); + int rc = k8s_inc_index(tmp, db_path); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + int stale = rc == 0 ? k8s_inc_count(db_path, v2) : -1; + int after = rc == 0 ? k8s_inc_count(db_path, v3) : -1; + th_rmtree(tmp); + ASSERT_EQ(before, 1); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_EQ(stale, 0); + ASSERT_EQ(after, 1); + PASS(); +} + +/* Helm's values as its templates see them (helm_template.c): go-yaml v2 + * scalars -- YAML 1.1 booleans, every number a float64 printed like Go's %v + * -- --set typing (int64, bool, null, a leading-zero string), coalesce where + * a null deletes, and a field of a missing map an error: no value. */ +TEST(helm_values_follow_helm_semantics) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + ASSERT_NOT_NULL(doc); + static const char src[] = "tag: 1.10\nbig: 20231010\nm: 1000000\nyes_: yes\noff_: Off\n" + "nul: ~\nq: \"1.10\"\noct: 0755\nkeep: k\ngone: g\n"; + yyjson_mut_val *v = cbm_helm_values_parse(doc, src, strlen(src)); + ASSERT_NOT_NULL(v); + char buf[64]; + ASSERT_TRUE(cbm_helm_scalar_text(yyjson_mut_obj_get(v, "tag"), buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "1.1"); + ASSERT_TRUE(cbm_helm_scalar_text(yyjson_mut_obj_get(v, "big"), buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "2.023101e+07"); + ASSERT_TRUE(cbm_helm_scalar_text(yyjson_mut_obj_get(v, "m"), buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "1e+06"); + ASSERT_TRUE(cbm_helm_scalar_text(yyjson_mut_obj_get(v, "oct"), buf, sizeof(buf))); + ASSERT_STR_EQ(buf, "493"); + ASSERT_TRUE(yyjson_mut_is_true(yyjson_mut_obj_get(v, "yes_"))); + ASSERT_TRUE(yyjson_mut_is_false(yyjson_mut_obj_get(v, "off_"))); + ASSERT_TRUE(yyjson_mut_is_null(yyjson_mut_obj_get(v, "nul"))); + ASSERT_STR_EQ(yyjson_mut_get_str(yyjson_mut_obj_get(v, "q")), "1.10"); + ASSERT_TRUE(cbm_helm_values_set(doc, v, "a\\.b.c[1]", "007", false)); + ASSERT_TRUE(cbm_helm_values_set(doc, v, "n", "42", false)); + ASSERT_TRUE(cbm_helm_values_set(doc, v, "s", "42", true)); + yyjson_mut_val *ab = yyjson_mut_obj_get(v, "a.b"); + ASSERT_NOT_NULL(ab); + yyjson_mut_val *c = yyjson_mut_obj_get(ab, "c"); + ASSERT_TRUE(yyjson_mut_is_arr(c)); + ASSERT_TRUE(yyjson_mut_is_null(yyjson_mut_arr_get(c, 0))); + ASSERT_STR_EQ(yyjson_mut_get_str(yyjson_mut_arr_get(c, 1)), "007"); + ASSERT_TRUE(yyjson_mut_is_int(yyjson_mut_obj_get(v, "n"))); + ASSERT_TRUE(yyjson_mut_is_str(yyjson_mut_obj_get(v, "s"))); + static const char over[] = "gone: null\nkeep: x\nnew: 1\n"; + yyjson_mut_val *o = cbm_helm_values_parse(doc, over, strlen(over)); + yyjson_mut_val *m = cbm_helm_values_coalesce(doc, v, o); + ASSERT_NULL(yyjson_mut_obj_get(m, "gone")); + ASSERT_STR_EQ(yyjson_mut_get_str(yyjson_mut_obj_get(m, "keep")), "x"); + ASSERT_NOT_NULL(yyjson_mut_obj_get(m, "new")); + ASSERT_STR_EQ(yyjson_mut_get_str(yyjson_mut_obj_get(v, "keep")), "k"); /* base untouched */ + /* a template: a missing last key renders "", a field of it is an error */ + static const char tpl[] = "image: \"{{ .Values.keep }}:{{ .Values.none }}\"\n" + "image: \"{{ .Values.none.repo }}\"\n"; + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_4K); + cbm_helm_probe_t probes[2] = {{.line = 1}, {.line = 2}}; + cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, NULL, NULL, probes, 2); + ASSERT_EQ(probes[0].n, 1); + ASSERT_STR_EQ(probes[0].image[0], "k:"); + ASSERT_EQ(probes[1].n, 1); + ASSERT_NULL(probes[1].image[0]); + cbm_arena_destroy(&a); + yyjson_mut_doc_free(doc); + PASS(); +} + /* A compose project's .env feeds its compose file (variables in image, * build): changing it is no closure repair -- the unchanged compose file's * services would keep the old image. It rebuilds; no stale image remains. */ @@ -23930,6 +24060,9 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_env_edit_replaces_scoped_envvars); RUN_TEST(incremental_image_split_change_rebuilds); RUN_TEST(incremental_helm_values_edit_keeps_chart_edge); + RUN_TEST(incremental_werf_values_edit_keeps_chart_edge); + RUN_TEST(incremental_release_values_edit_rebuilds); + RUN_TEST(helm_values_follow_helm_semantics); RUN_TEST(incremental_compose_dotenv_change_rebuilds); RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges); RUN_TEST(incremental_second_build_declarer_keeps_the_edge); From 7528ec06fe7da5ab703e7c802624fef88285ae37 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 17 Oct 2026 22:33:37 +0200 Subject: [PATCH 04/19] test(pipeline): bind the Go selector guard on the parallel resolver Review follow-up for #1907 (asks from the 2026-09-28 review): - pipeline_go_receiver_parallel_suppresses_weak_method_edge: parallel twin of pipeline_go_receiver_suppresses_weak_method_edge. The guard is wired into pass_parallel.c as well as pass_calls.c, but the four-file fixture only runs the sequential resolver, so the parallel hunk was bound by no test. The fixture is shared (write_go_receiver_fixture) and padded with 52 Go files past MIN_FILES_FOR_PARALLEL, with CBM_WORKERS=4, in the shape of the Python parallel twin. - registry.c: cbm_go_suppress_weak_method_match builds on the shared weak_short_name_strategy list and narrows it in two places only (field_type_hint kept, unique_name only when penalized). Behaviour is unchanged; weak_call_guards_share_one_drop_list now binds the Go guard to the shared list too. - test_extraction.c: the flag-exempt comment names the current flag set (Perl, Python, TS/JS/TSX and Go) and Rust as the exempt language. Refs #1906 Signed-off-by: Martin Vogel --- src/pipeline/registry.c | 24 +++++++------ tests/test_extraction.c | 4 +-- tests/test_pipeline.c | 78 +++++++++++++++++++++++++++++++++++++---- tests/test_registry.c | 10 ++++++ 4 files changed, 97 insertions(+), 19 deletions(-) diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index cbfa4ec5d..bfc2e330e 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -750,19 +750,23 @@ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_b bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *strategy, double confidence) { - if (!is_go || !is_method || !strategy || !strategy[0]) { - return false; - } /* Go analog of the TS/JS guard above, same failure class: a selector call * whose receiver the Go LSP could not type reaches the registry and a bare * short-name strategy binds it to an arbitrary same-named project symbol * (`f.Close()` on an os.File -> a project `Close`, suffix_match over 15 - * candidates). Unlike the TS/JS list, field_type_hint is KEPT: a Go struct - * field carries a declared type, so the parallel resolver's field-type - * hint is receiver-aware for Go (lrp_go_s8_field_type_hint), not a - * heuristic. */ - if (strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "fuzzy") == 0) { - return true; + * candidates). Built on the shared drop-list so "weak" means one thing + * for every guard; Go narrows it in two places only. */ + if (!is_go || !is_method || !weak_short_name_strategy(strategy)) { + return false; + } + /* field_type_hint is KEPT for Go: a Go struct field carries a declared + * type, so the parallel resolver's field-type hint is receiver-aware for + * Go (lrp_go_s8_field_type_hint), not a heuristic. */ + if (strcmp(strategy, "field_type_hint") == 0) { + return false; + } + if (strcmp(strategy, "unique_name") != 0) { + return true; /* suffix_match, fuzzy */ } /* unique_name is dropped only when PENALIZED: resolve_name_lookup scales * CONF_UNIQUE_NAME by DEFAULT_CONFIDENCE exactly when the lone candidate @@ -772,7 +776,7 @@ bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *s * file has no imports, e.g. a same-package call) and must be kept — * dropping it kills genuinely-typed lone-candidate calls that never * enter the field-type-hint upgrade (candidate_count == 1). */ - return strcmp(strategy, "unique_name") == 0 && confidence < CONF_UNIQUE_NAME; + return confidence < CONF_UNIQUE_NAME; } /* Import-binding counterpart of the parameter guard above (#2127). An import diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 41a796ce8..13b03aa37 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -6886,8 +6886,8 @@ TEST(extract_perl_method_call_flags_is_method) { PASS(); } -/* Languages OUTSIDE the is_method flag set (only Perl and TS/JS/TSX set it) must - * be unaffected: a Go method call never sets is_method. */ +/* Languages OUTSIDE the is_method flag set (Perl, Python, TS/JS/TSX and Go set + * it) must be unaffected: a Rust method call never sets is_method. */ TEST(extract_flag_exempt_method_call_not_flagged_is_method) { /* Rust is flag-exempt: only Perl, Python, TS/JS and Go set is_method. * Guards the blast radius of the receiver-aware flags for every other diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 4da054b14..d413aa9bb 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8650,13 +8650,11 @@ TEST(pipeline_html_embedded_member_call_stays_unbound) { PASS(); } -TEST(pipeline_go_receiver_suppresses_weak_method_edge) { - char tmp[256]; - snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_XXXXXX"); - if (!cbm_mkdtemp(tmp)) { - FAIL("tmpdir"); - } - +/* Fixture for the #1906 Go selector-guard probes (sequential and parallel + * twins). pad_files > 0 adds filler Go files to push the index over + * MIN_FILES_FOR_PARALLEL, because the guard is wired into both resolvers + * (pass_calls.c and pass_parallel.c) and each needs its own probe. */ +static void write_go_receiver_fixture(const char *tmp, int pad_files) { /* go.mod makes project imports resolvable — real Go repos always have one, * and import reachability (the unique_name penalty) depends on it. */ write_temp_file(tmp, "go.mod", "module example.com/myapp\n\ngo 1.22\n"); @@ -8715,6 +8713,22 @@ TEST(pipeline_go_receiver_suppresses_weak_method_edge) { "func CallsLocal() int { return localHelper() }\n" "\n" "func UsesUtil() string { return util.Tag() }\n"); + for (int i = 0; i < pad_files; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "filler/filler%d.go", i); + snprintf(body, sizeof(body), "package filler\n\nfunc Filler%d() int { return %d }\n", i, i); + write_temp_file(tmp, name, body); + } +} + +TEST(pipeline_go_receiver_suppresses_weak_method_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_go_receiver_fixture(tmp, 0); char db_path[512]; snprintf(db_path, sizeof(db_path), "%s/go_recv.db", tmp); @@ -8743,6 +8757,55 @@ TEST(pipeline_go_receiver_suppresses_weak_method_edge) { PASS(); } +/* Parallel twin of the test above. The guard is wired into pass_parallel.c as + * well as pass_calls.c, but the four-file fixture only ever runs the + * sequential resolver, so the parallel hunk was bound by no test. Real + * repositories take the parallel path: >= MIN_FILES_FOR_PARALLEL (50) files + * plus CBM_WORKERS=4 force it, the shape of + * pipeline_python_receiver_parallel_suppresses_weak_method_edges. Same + * both-directions pin as the sequential test. */ +TEST(pipeline_go_receiver_parallel_suppresses_weak_method_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_par_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_go_receiver_fixture(tmp, 52); + + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", "4", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_recv_par.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* NEGATIVE: the stdlib *os.File Close must not bind the project method. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "FileLen", "Close")); + /* POSITIVE: typed same-package, bare local and import-qualified calls + * survive the parallel resolver too. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "Boot", "Close")); + ASSERT_TRUE(cross_file_call_exists(s, project, "CallsLocal", "localHelper")); + ASSERT_TRUE(cross_file_call_exists(s, project, "UsesUtil", "Tag")); + + cbm_store_close(s); + cbm_pipeline_free(p); + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + PASS(); +} + /* Fixture for the #1928 cross-language reference-guard probes (sequential and * parallel twins). pad_files > 0 adds filler files to push the index over the * parallel-pipeline threshold, since USAGE/WRITES/READS have one resolver per @@ -23707,6 +23770,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_receiver_keeps_specific_unique_name_member_call); RUN_TEST(pipeline_html_embedded_member_call_stays_unbound); RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_receiver_parallel_suppresses_weak_method_edge); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); RUN_TEST(pipeline_rust_std_receiver_never_binds_project_method); diff --git a/tests/test_registry.c b/tests/test_registry.c index e25ba369e..a6fbd6201 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1673,6 +1673,16 @@ TEST(weak_call_guards_share_one_drop_list) { strategies[i], member, binding); } ASSERT_EQ(member, binding); + /* The Go selector guard builds on the same list and narrows it in + * exactly one place at a penalized confidence (0.75 * 0.5): it keeps + * field_type_hint. Any other drift from the shared list fails here. */ + bool go = cbm_go_suppress_weak_method_match(true, true, strategies[i], 0.375); + bool go_expected = member && strcmp(strategies[i], "field_type_hint") != 0; + if (go != go_expected) { + printf(" Go drop-list divergence on strategy \"%s\": go=%d expected=%d\n", + strategies[i], go, go_expected); + } + ASSERT_EQ(go, go_expected); } PASS(); } From 87cafb9ef8723b42e6348a9b2be864575a476699 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 17 Oct 2026 22:40:55 +0200 Subject: [PATCH 05/19] fix(extract): bump the index format for receiver-qualified Go method QNs Review follow-up for #1913 (asks from the 2026-09-28 review), plus one finding from the review of the updated head: - store.h: CBM_INDEX_FORMAT_VERSION 3 -> 4. Every Go method QN changes (pkg.method -> pkg.Recv.method), so an index built before this change and refreshed incrementally would mix flat and receiver-qualified method nodes, with the old fused nodes keeping their stale edges. The bump routes such an index through one full rebuild. - pipeline_go_method_caller_keeps_lsp_join: the by-name assertion would also pass for a binding onto the decoy other.Decoy.helper. The test now pins the edge by QN suffix on both ends (.svc.Repo.Install -> .svc.Repo.helper) with an lsp_* strategy, and asserts no edge to the decoy. - extract_defs.c: the receiver-qualified def QN is Go-only. The templ grammar is Go-derived and reaches the same receiver branch, but its call-scope QN (compute_func_qn) stays flat, so a templ method would get a def QN no call names and its calls would degrade to the File node. extract_templ_method_qn_matches_call_scope pins def QN == call scope. - pass_semantic.c: the check_go_class_implements comment no longer describes Go method QNs as flat. Refs #1909 Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 9 ++++++-- src/pipeline/pass_semantic.c | 6 ++--- src/store/store.h | 6 +++-- tests/test_extraction.c | 43 ++++++++++++++++++++++++++++++++++++ tests/test_pipeline.c | 42 +++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 9bf027d17..176d0a0b8 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -6696,8 +6696,13 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec * call-scope side (compute_func_qn in extract_unified.c) mirrors * this formula, and go_lsp consumers read the def QN and * parent_class (receiver_type) verbatim, so resolution joins - * stay exact. */ - def.qualified_name = cbm_arena_sprintf(a, "%s.%s", def.parent_class, name); + * stay exact. Go only: templ's Go-derived grammar reaches this + * branch too, but its call-scope QN (compute_func_qn) stays flat, + * and a def QN the calls do not share would orphan every call + * inside a templ method. */ + if (ctx->language == CBM_LANG_GO) { + def.qualified_name = cbm_arena_sprintf(a, "%s.%s", def.parent_class, name); + } } } diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 186b2dfdc..bc23e0d26 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -266,9 +266,9 @@ static int check_go_class_implements(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_nod } /* Resolve the struct's methods two ways and use whichever finds them: * (a) the struct's DEFINES_METHOD edges, matched by method NAME — the real - * pipeline path, where Go receiver methods carry a flat QN (e.g. - * "pkg.Area" rather than "pkg.Circle.Area"), so QN-string - * reconstruction would miss; and + * pipeline path. Go method QNs are receiver-qualified since #1913 + * ("pkg.Circle.Area"), so (b) matches them as well, but the edges + * stay the primary join; and * (b) the reconstructed QN "." — used by tests and any * extractor that does emit class-qualified method QNs without * DEFINES_METHOD edges from the class. */ diff --git a/src/store/store.h b/src/store/store.h index 803df81d8..0f291daf9 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -32,8 +32,10 @@ typedef struct cbm_store_read_scope cbm_store_read_scope_t; * Dockerfile. keeps its suffix in its QNs. * 3 Image identity invents no registry: a reference without a host is * __image__, registry "unknown" -- no longer - * __image__docker.io/[library/]. */ -#define CBM_INDEX_FORMAT_VERSION 3 + * __image__docker.io/[library/]. + * 4 #1913: Go method QNs are receiver-qualified (pkg.Recv.method, generic + * receivers unwrapped); free functions stay pkg.func. */ +#define CBM_INDEX_FORMAT_VERSION 4 #define CBM_STORE_CANCELLED (-3) #define CBM_STORE_SCAN_LIMIT (-4) #define CBM_STORE_CALLBACK_ERR (-5) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 8dfc5e6e1..65919f6c3 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7229,6 +7229,48 @@ TEST(extract_go_method_receiver_qualified_qn) { PASS(); } +/* Templ shares Go's method grammar (method_declaration with a receiver), but + * only Go method QNs are receiver-qualified (#1913); templ's call-scope QN + * stays flat. The method's def QN and the enclosing-function QN of a call + * inside it must agree, or the call has no source node and degrades to the + * File node. */ +TEST(extract_templ_method_qn_matches_call_scope) { + CBMFileResult *r = extract("package views\n" + "\n" + "type Page struct{}\n" + "\n" + "func (p Page) Title() string {\n" + "\treturn helper()\n" + "}\n" + "\n" + "func helper() string { return \"x\" }\n" + "\n" + "templ hello(name string) {\n" + "\t
{ name }
\n" + "}\n", + CBM_LANG_TEMPL, "t", "x.templ"); + ASSERT_NOT_NULL(r); + const char *title_qn = NULL; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->name && d->qualified_name && strcmp(d->name, "Title") == 0) { + title_qn = d->qualified_name; + } + } + const char *scope = NULL; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + if (c->callee_name && strcmp(c->callee_name, "helper") == 0) { + scope = c->enclosing_func_qn; + } + } + ASSERT_NOT_NULL(title_qn); + ASSERT_NOT_NULL(scope); + ASSERT_STR_EQ(scope, title_qn); + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -12301,6 +12343,7 @@ SUITE(extraction) { RUN_TEST(extract_python_bare_call_flag_is_depth_independent); RUN_TEST(extract_go_selector_call_flags_is_method); RUN_TEST(extract_go_method_receiver_qualified_qn); + RUN_TEST(extract_templ_method_qn_matches_call_scope); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 04925d6dc..bb4cd174d 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8650,6 +8650,41 @@ TEST(pipeline_html_embedded_member_call_stays_unbound) { PASS(); } +/* CALLS edges whose source and target QNs end with the given suffixes and + * whose serialized properties contain `props_fragment` (NULL: any edge). QN + * suffixes pin the exact node, not just a same-named one. */ +static int go_calls_between(cbm_store_t *s, const char *project, const char *caller_suffix, + const char *callee_suffix, const char *props_fragment) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count); + size_t cs = strlen(caller_suffix); + size_t ts = strlen(callee_suffix); + int hits = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK && + src.qualified_name && tgt.qualified_name) { + size_t sl = strlen(src.qualified_name); + size_t tl = strlen(tgt.qualified_name); + if (sl >= cs && tl >= ts && strcmp(src.qualified_name + sl - cs, caller_suffix) == 0 && + strcmp(tgt.qualified_name + tl - ts, callee_suffix) == 0 && + (!props_fragment || + (edges[i].properties_json && strstr(edges[i].properties_json, props_fragment)))) { + hits++; + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) { + cbm_store_free_edges(edges, edge_count); + } + return hits; +} + TEST(pipeline_go_method_caller_keeps_lsp_join) { /* #1909: receiver-qualifying method QNs moves the def/textual-call QN to * package.Type.method — go_lsp's enclosing-function QN must move with it @@ -8695,6 +8730,13 @@ TEST(pipeline_go_method_caller_keeps_lsp_join) { cbm_store_t *s = cbm_store_open_path(db_path); ASSERT_NOT_NULL(s); ASSERT_TRUE(cross_file_call_exists(s, project, "Install", "helper")); + /* Pinned to the receiver-qualified target in svc/helper.go and to an LSP + * strategy: a name-only binding, or one onto other.Decoy.helper, would + * still satisfy the by-name check above. */ + ASSERT_GTE(go_calls_between(s, project, ".svc.Repo.Install", ".svc.Repo.helper", + "\"strategy\":\"lsp_"), + 1); + ASSERT_EQ(go_calls_between(s, project, ".svc.Repo.Install", ".other.Decoy.helper", NULL), 0); cbm_store_close(s); cbm_pipeline_free(p); th_rmtree(tmp); From db8d547ae1acb444f69cf08a5f81d4e8ec5e83d9 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 17 Oct 2026 23:37:31 +0200 Subject: [PATCH 06/19] fix(extract): keep both Go method QN sides flat when the type QN is NULL cbm_fqn_compute_source_lang returns NULL only when the module QN cannot be built (arena exhaustion). The receiver-qualified branches then passed that NULL to "%s". Both sides now fall back to the flat QN together, so the def and its calls still agree. Allocation-failure path only; no change on any normal run. (Raised in our security review of the updated head.) Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 2 +- internal/cbm/extract_unified.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 176d0a0b8..1257945a5 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -6700,7 +6700,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec * branch too, but its call-scope QN (compute_func_qn) stays flat, * and a def QN the calls do not share would orphan every call * inside a templ method. */ - if (ctx->language == CBM_LANG_GO) { + if (ctx->language == CBM_LANG_GO && def.parent_class) { def.qualified_name = cbm_arena_sprintf(a, "%s.%s", def.parent_class, name); } } diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 009a91194..aac32e786 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -1109,7 +1109,11 @@ static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLan if (recv_type && recv_type[0]) { const char *type_qn = cbm_fqn_compute_source_lang( ctx->arena, ctx->project, ctx->rel_path, recv_type, ctx->language); - return cbm_arena_sprintf(ctx->arena, "%s.%s", type_qn, name); + /* NULL only on allocation failure: fall through to the flat + * QN, as the def side does (extract_defs.c). */ + if (type_qn) { + return cbm_arena_sprintf(ctx->arena, "%s.%s", type_qn, name); + } } } } From e554a2ab58ca954c2e9f46ba3fe9d340d1bcf06d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 18 Oct 2026 16:54:36 +0200 Subject: [PATCH 07/19] fix(go): no field-type hint for Go; it is a name guess, par-only The parallel resolver re-picks a multi-candidate registry match through try_field_type_hint. That helper capitalizes the receiver VARIABLE name and accepts any candidate whose QN contains it as a substring: `t.Run` on a *testing.T became `T` and bound a project `TBench.Run`. The sequential resolver has no such step. The Go selector guard treated field_type_hint as receiver-aware and kept it, so the same weak pick it drops in sequential mode survived in parallel mode under a new name. On caddy + prometheus, parallel and sequential CALLS diverged by 1,684 (this branch) and 5,560 (with receiver-qualified method QNs, which put type names into more QNs). At base the gap is 84. SCIP precision of Go field_type_hint is 0.016 at base. Go no longer takes the hint. The guard keeps field_type_hint on the shared weak list, and the fixture of both twins now pins the `t.Run` case. A typed field call (lrp_go_s8, `s.repo.Find()`) resolves through the Go LSP and is unaffected. Signed-off-by: Martin Vogel --- src/pipeline/pass_parallel.c | 10 +++++++++- src/pipeline/registry.c | 14 ++++++-------- tests/test_pipeline.c | 28 ++++++++++++++++++++++++++++ tests/test_registry.c | 15 ++++++++------- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 37bdc26d4..56a85e8a2 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3298,7 +3298,15 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB } _rc_t0 = extract_now_ns(); - try_field_type_hint(rc, &res, call->callee_name, source_node->id); + /* Not for Go (F-G9): the hint matches the capitalized receiver + * VARIABLE name as a substring of a candidate QN (`t.Run` -> any + * `...T...Run`), with no type evidence. The sequential resolver has no + * such step, so for Go it produced parallel-only edges (par/seq CALLS + * diverged), and the Go selector guard could not see them as the weak + * pick they are. SCIP precision of Go field_type_hint: 0.016. */ + if (lang != CBM_LANG_GO) { + try_field_type_hint(rc, &res, call->callee_name, source_node->id); + } /* Cross-language veto — the same predicate as pass_calls.c, applied * AFTER the field-type hint so the hint may still re-pick a compatible * candidate first, and BEFORE the empty-resolution fallbacks below so a diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index bfc2e330e..9b5165a27 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -755,18 +755,16 @@ bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *s * short-name strategy binds it to an arbitrary same-named project symbol * (`f.Close()` on an os.File -> a project `Close`, suffix_match over 15 * candidates). Built on the shared drop-list so "weak" means one thing - * for every guard; Go narrows it in two places only. */ + * for every guard; Go narrows it in one place only (unique_name below). + * field_type_hint stays weak: it is a receiver-NAME substring match, not + * a declared-type lookup (the parallel resolver no longer produces it for + * Go; a typed field call such as lrp_go_s8's `s.repo.Find()` resolves + * through the Go LSP). */ if (!is_go || !is_method || !weak_short_name_strategy(strategy)) { return false; } - /* field_type_hint is KEPT for Go: a Go struct field carries a declared - * type, so the parallel resolver's field-type hint is receiver-aware for - * Go (lrp_go_s8_field_type_hint), not a heuristic. */ - if (strcmp(strategy, "field_type_hint") == 0) { - return false; - } if (strcmp(strategy, "unique_name") != 0) { - return true; /* suffix_match, fuzzy */ + return true; /* suffix_match, field_type_hint, fuzzy */ } /* unique_name is dropped only when PENALIZED: resolve_name_lookup scales * CONF_UNIQUE_NAME by DEFAULT_CONFIDENCE exactly when the lone candidate diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index d413aa9bb..6d04ad327 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8713,6 +8713,30 @@ static void write_go_receiver_fixture(const char *tmp, int pad_files) { "func CallsLocal() int { return localHelper() }\n" "\n" "func UsesUtil() string { return util.Tag() }\n"); + /* F-G9: two project methods named Run, one under a directory whose QN + * carries an uppercase T. `t.Run` on a *testing.T must bind neither. The + * parallel resolver field-type hint matched the capitalized receiver + * variable ("T") as a QN substring and produced a par-only edge. */ + write_temp_file(tmp, "TBench/bench.go", + "package tbench\n" + "\n" + "type Bench struct{}\n" + "\n" + "func (b *Bench) Run(name string) bool { return name != \"\" }\n"); + write_temp_file(tmp, "worker/worker.go", + "package worker\n" + "\n" + "type Worker struct{}\n" + "\n" + "func (w *Worker) Run(name string) bool { return false }\n"); + write_temp_file(tmp, "hash/hash_test.go", + "package hash\n" + "\n" + "import \"testing\"\n" + "\n" + "func TestSubRun(t *testing.T) {\n" + "\tt.Run(\"a\", func(t *testing.T) {})\n" + "}\n"); for (int i = 0; i < pad_files; i++) { char name[64]; char body[128]; @@ -8750,6 +8774,8 @@ TEST(pipeline_go_receiver_suppresses_weak_method_edge) { /* (4) The import-qualified cross-package call survives (import-aware * strategies are outside the drop-list). */ ASSERT_TRUE(cross_file_call_exists(s, project, "UsesUtil", "Tag")); + /* (5) F-G9: the stdlib *testing.T Run binds no project Run. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "TestSubRun", "Run")); cbm_store_close(s); cbm_pipeline_free(p); @@ -8793,6 +8819,8 @@ TEST(pipeline_go_receiver_parallel_suppresses_weak_method_edge) { ASSERT_TRUE(cross_file_call_exists(s, project, "Boot", "Close")); ASSERT_TRUE(cross_file_call_exists(s, project, "CallsLocal", "localHelper")); ASSERT_TRUE(cross_file_call_exists(s, project, "UsesUtil", "Tag")); + /* F-G9: the parallel-only field-type hint must not bind `t.Run` either. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "TestSubRun", "Run")); cbm_store_close(s); cbm_pipeline_free(p); diff --git a/tests/test_registry.c b/tests/test_registry.c index a6fbd6201..90ab69170 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1673,11 +1673,12 @@ TEST(weak_call_guards_share_one_drop_list) { strategies[i], member, binding); } ASSERT_EQ(member, binding); - /* The Go selector guard builds on the same list and narrows it in - * exactly one place at a penalized confidence (0.75 * 0.5): it keeps - * field_type_hint. Any other drift from the shared list fails here. */ + /* The Go selector guard builds on the same list. At a penalized + * confidence (0.75 * 0.5) it agrees with it exactly; its one narrowing + * (an unpenalized unique_name) does not apply here. Any drift from + * the shared list fails here. */ bool go = cbm_go_suppress_weak_method_match(true, true, strategies[i], 0.375); - bool go_expected = member && strcmp(strategies[i], "field_type_hint") != 0; + bool go_expected = member; if (go != go_expected) { printf(" Go drop-list divergence on strategy \"%s\": go=%d expected=%d\n", strategies[i], go, go_expected); @@ -1697,6 +1698,9 @@ TEST(go_suppress_drops_weak_selector_matches) { ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "suffix_match", 0.11)); ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "fuzzy", 0.9)); ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "unique_name", 0.375)); + /* field_type_hint is a receiver-NAME substring match, not type evidence + * (F-G9): weak for Go like the shared list. */ + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "field_type_hint", 0.85)); PASS(); } @@ -1706,9 +1710,6 @@ TEST(go_suppress_keeps_typed_and_import_aware_matches) { * lone-candidate call never enters the field-type-hint upgrade, so it must * survive (lrp_go_s8_field_type_hint). */ ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "unique_name", 0.75)); - /* field_type_hint is receiver-aware for Go — struct fields carry declared - * types (lrp_go_s8_field_type_hint) — so it stays, unlike the TS/JS list. */ - ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "field_type_hint", 0.85)); ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "same_module", 0.9)); ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "import_map", 0.95)); ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "import_map_suffix", 0.9)); From 4cea54f75835cadd402d60733c9310113e11a4a6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 18 Oct 2026 21:08:20 +0200 Subject: [PATCH 08/19] fix(deploy): bound Helm rendering on hostile input, no partial or wrong release images Review round 1 of #2677. Every case ends in an explicit unknown, never in a hang, an OOM, a partial or a wrong image. Bounds (helm_template.c): - A per-render step and output budget (10,000,000 steps, 64 MiB): a range over a huge int, a huge indent, defines that include each other in a fan stop the render; every probe of the template is UNKNOWN, also one captured before, and the trip is logged (helm.render.budget). Counted, never timed. On the 20 infra corpora no render reaches it. - go-yaml v2's alias limit: values whose decodes are mostly alias expansion (a billion laughs) do not parse, as in Helm. - --set: an index past strvals' MaxIndex (65536) or a key longer than the parser holds is no value (the release is unknown), never a padded list or a split key. Reads (pass_k8s.c, pass_terraform.c): a values file, Chart.yaml, kustomize valuesFile or Terraform file() named by path is read only when it is a discovered file of the repository -- never through a symlink out of it -- with the size cap and tree-sitter padding of every other read. A chart file that is there but not read makes the chart's values unknown, not empty. Wrong images: - Terraform number sets reach the chart as Terraform's exact decimal (1.10 -> "1.1"), not %.17g. - A release with a value source not read -- Flux valuesFrom, valuesFiles, postRenderers; kustomize additionalValuesFiles, a valuesMerge other than override, a valuesFile not read; Terraform dynamic "set" -- has no image of its own. - An aliased subchart renders with .Chart.Name its alias. - An UNKNOWN if / with / range makes what its bodies assign UNKNOWN and leaves the loop a break in them would; set / unset / merge poison the map they change (merge: the maps within too). A change that may reach values another part holds taints every later capture; one of a new value (fromYaml, deepCopy, an empty dict, a part of one) does not. Marks and secrets: - The chart edge reads default_of as JSON (list paths kept), builds its properties to size, and writes no edge whose mark it could not carry whole. - Build arguments with a secret (by name or value) are never kept; build edge properties and values-path marks are never cut. Tests: one per bound and per wrong-image case, each RED on d21949ef by its assertion; a Helm chart joins the Kubernetes complexity fixture. Signed-off-by: Martin Vogel --- src/pipeline/helm_template.c | 839 +++++++++++++++++- src/pipeline/helm_template.h | 6 +- src/pipeline/pass_ci.c | 19 +- src/pipeline/pass_k8s.c | 230 +++-- src/pipeline/pass_shell.c | 28 +- src/pipeline/pass_terraform.c | 186 +++- .../complexity/Kubernetes/chart/Chart.yaml | 4 + .../Kubernetes/chart/templates/_helpers.tpl | 4 + .../Kubernetes/chart/templates/deploy.yaml | 14 + .../complexity/Kubernetes/chart/values.yaml | 12 + tests/test_grammar_probe_f.c | 355 ++++++++ tests/test_pipeline.c | 234 +++++ 12 files changed, 1780 insertions(+), 151 deletions(-) create mode 100644 tests/fixtures/complexity/Kubernetes/chart/Chart.yaml create mode 100644 tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl create mode 100644 tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml create mode 100644 tests/fixtures/complexity/Kubernetes/chart/values.yaml diff --git a/src/pipeline/helm_template.c b/src/pipeline/helm_template.c index 79293e638..fa516ae00 100644 --- a/src/pipeline/helm_template.c +++ b/src/pipeline/helm_template.c @@ -23,6 +23,7 @@ #include "foundation/constants.h" #include "foundation/hash_table.h" #include "foundation/json_writer.h" /* cbm_json_writer_allocator */ +#include "foundation/log.h" #include "foundation/mem_core.h" #include "cbm.h" #include "lang_specs.h" @@ -34,7 +35,17 @@ #include #include -enum { HELM_MAX_NESTING = 1000 }; /* Helm engine's recursionMaxNums */ +enum { HELM_MAX_NESTING = 1000 }; /* Helm engine's recursionMaxNums */ +enum { HELM_MAX_SET_INDEX = 65536 }; /* Helm strvals' MaxIndex for `--set a[n]` */ + +/* A render doing more than this -- steps (an action, an evaluation, a range + * iteration) or bytes written to any sink -- is one Helm never finishes + * either (a range over a huge int, defines that call each other in a fan): + * it stops, and every probe of the template is UNKNOWN. Counted, never timed; + * far above any real chart (the 20 infra corpora and the helm oracle charts + * never reach it). */ +enum { HELM_MAX_STEPS = 10000000 }; +static const size_t HELM_MAX_OUTPUT = (size_t)CBM_SZ_64 * CBM_SZ_1K * CBM_SZ_1K; /* 64 MiB */ /* ── Values: YAML -> JSON ─────────────────────────────────────────── */ @@ -42,6 +53,7 @@ typedef struct { const char *name; size_t len; yyjson_mut_val *val; + long decodes; /* the nodes its value took to decode, its aliases expanded */ } hy_anchor_t; typedef struct { @@ -51,8 +63,37 @@ typedef struct { int nanchors; int cap; bool unknown; + long decodes; /* nodes decoded, alias expansions included (go-yaml decodeCount) */ + long alias_decodes; /* of those, the ones an alias expanded (aliasCount) */ } hy_t; +/* go-yaml v2 (decode.go): a document whose decodes come mostly from alias + * expansion -- a billion laughs -- is an error, so Helm reads no values. The + * share allowed falls from 99% below 400,000 decodes to 10% above 4,000,000. */ +enum { + HY_ALIAS_MIN_ALIASES = 100, + HY_ALIAS_MIN_DECODES = 1000, + HY_ALIAS_RANGE_LOW = 400000, + HY_ALIAS_RANGE_HIGH = 4000000 +}; +static const double HY_ALIAS_RATIO_SMALL = 0.99; +static const double HY_ALIAS_RATIO_LARGE = 0.10; + +static bool hy_excessive_aliasing(const hy_t *y) { + if (y->alias_decodes <= HY_ALIAS_MIN_ALIASES || y->decodes <= HY_ALIAS_MIN_DECODES) { + return false; + } + double allowed = HY_ALIAS_RATIO_SMALL; + if (y->decodes >= HY_ALIAS_RANGE_HIGH) { + allowed = HY_ALIAS_RATIO_LARGE; + } else if (y->decodes > HY_ALIAS_RANGE_LOW) { + allowed = HY_ALIAS_RATIO_SMALL - (HY_ALIAS_RATIO_SMALL - HY_ALIAS_RATIO_LARGE) * + ((double)(y->decodes - HY_ALIAS_RANGE_LOW) / + (double)(HY_ALIAS_RANGE_HIGH - HY_ALIAS_RANGE_LOW)); + } + return (double)y->alias_decodes / (double)y->decodes > allowed; +} + static bool hy_type(TSNode n, const char *t) { return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; } @@ -63,7 +104,7 @@ static const char *hy_text(const hy_t *y, TSNode n, size_t *len) { return y->src + b; } -static void hy_anchor(hy_t *y, const char *name, size_t len, yyjson_mut_val *v) { +static void hy_anchor(hy_t *y, const char *name, size_t len, yyjson_mut_val *v, long decodes) { if (y->nanchors == y->cap) { int cap = y->cap ? y->cap * PAIR_LEN : CBM_SZ_16; hy_anchor_t *g = cbm_realloc(CBM_MEM_CLASS_OTHER, y->anchors, (size_t)cap * sizeof(*g)); @@ -74,12 +115,19 @@ static void hy_anchor(hy_t *y, const char *name, size_t len, yyjson_mut_val *v) y->anchors = g; y->cap = cap; } - y->anchors[y->nanchors++] = (hy_anchor_t){name, len, v}; + y->anchors[y->nanchors++] = (hy_anchor_t){name, len, v, decodes}; } +/* An alias: its anchor's value decoded again (a copy), as go-yaml counts it. */ static yyjson_mut_val *hy_alias(hy_t *y, const char *name, size_t len) { for (int i = y->nanchors - SKIP_ONE; i >= 0; i--) { if (y->anchors[i].len == len && memcmp(y->anchors[i].name, name, len) == 0) { + y->decodes += y->anchors[i].decodes; + y->alias_decodes += y->anchors[i].decodes; + if (hy_excessive_aliasing(y)) { + y->unknown = true; + return NULL; + } return yyjson_mut_val_mut_copy(y->doc, y->anchors[i].val); } } @@ -609,20 +657,24 @@ static yyjson_mut_val *hy_node(hy_t *y, TSNode n) { } } yyjson_mut_val *v = NULL; + long before = y->decodes; if (ts_node_is_null(content)) { + y->decodes++; v = yyjson_mut_null(y->doc); } else if (str_tag && hy_type(content, "plain_scalar")) { size_t l = 0; const char *s = hy_text(y, content, &l); + y->decodes++; v = yyjson_mut_strncpy(y->doc, s, l); } else { v = hy_node(y, content); } if (v && anchor) { - hy_anchor(y, anchor, al, v); + hy_anchor(y, anchor, al, v, y->decodes - before); } return v; } + y->decodes++; /* a mapping, sequence, scalar or alias node */ size_t l = 0; const char *s = hy_text(y, n, &l); if (strcmp(t, "block_mapping") == 0) { @@ -786,11 +838,15 @@ bool cbm_helm_values_set(yyjson_mut_doc *doc, yyjson_mut_val *root, const char * key[k++] = *p++; } key[k] = '\0'; + if (*p && *p != '.' && *p != '[') { + return false; /* a key longer than the buffer: never split into two */ + } long index = -SKIP_ONE; if (*p == '[') { char *end = NULL; index = strtol(p + SKIP_ONE, &end, 10); - if (!end || *end != ']' || index < 0) { + /* strvals: a negative index or one past MaxIndex is an error */ + if (!end || *end != ']' || index < 0 || index > HELM_MAX_SET_INDEX) { return false; } p = end + SKIP_ONE; @@ -1192,9 +1248,12 @@ struct hv { hv_map_t *m; hv_list_t *l; const char *path; /* the values path it holds; NULL: none */ + bool own; /* a new value holding nothing another part holds (fromYaml, deepCopy, + * a part of one; known or not): changing it changes nothing else */ }; static const hv_t HV_UNKNOWN = {.k = HV_UNK}; +static const hv_t HV_UNKNOWN_OWN = {.k = HV_UNK, .own = true}; static hv_t hv_nil(void) { return (hv_t){.k = HV_NIL}; @@ -1305,6 +1364,12 @@ typedef struct { hsink_t *out; /* NULL: the template's own output, which the probes read */ } hframe_t; +typedef struct { + const void *map; /* the hv_map_t / yyjson object changed */ + const char *path; /* its values path (NULL: none) */ + bool deep; /* merge: every map within it may have changed too */ +} hpoison_t; + typedef struct { CBMArena *a; yyjson_mut_doc *scratch; @@ -1315,6 +1380,16 @@ typedef struct { int nesting; bool brk; bool cont; + long steps; + size_t written; /* bytes appended to any sink */ + bool stop; /* over HELM_MAX_STEPS / HELM_MAX_OUTPUT: the render is UNKNOWN */ + /* maps a set / unset / merge changed in place (not modelled): what they + * hold is UNKNOWN from then on */ + hpoison_t *poison; + int npoison; + int pcap; + bool tainted; /* a change of unknown target: every capture ending later is UNKNOWN */ + CBMHashTable *mutates; /* define name -> it may change a map ((void *)1 / (void *)2) */ /* probes of the top-level template */ const char *top_src; size_t top_len; @@ -1333,10 +1408,32 @@ typedef struct { int tcap; } hst_t; +/* One step of work; false once the render is over its budget. */ +static bool hst_step(hst_t *st) { + if (!st->stop && ++st->steps > HELM_MAX_STEPS) { + st->stop = true; + } + return !st->stop; +} + +/* n more bytes of output; false (and the render stopped) past the budget. */ +static bool hst_write(hst_t *st, size_t n) { + if (st->stop || n > HELM_MAX_OUTPUT - st->written) { + st->stop = true; + return false; + } + st->written += n; + return true; +} + static void hs_append(hst_t *st, hsink_t *s, const char *p, size_t n) { if (s->unk || n == 0) { return; } + if (!hst_write(st, n)) { + s->unk = true; + return; + } if (s->n + n + SKIP_ONE > s->cap) { size_t cap = (s->cap ? s->cap : CBM_SZ_64) * PAIR_LEN; while (cap < s->n + n + SKIP_ONE) { @@ -1442,6 +1539,85 @@ static hvar_t *hst_var(hst_t *st, const hframe_t *fr, const char *name, size_t n return NULL; } +/* Has a set / unset / merge changed map v (or, for a merge, a map holding + * it)? Then what v holds is UNKNOWN. */ +static bool hv_poisoned(const hst_t *st, const hv_t *v) { + const void *map = v->k == HV_MAP ? (const void *)v->m + : v->k == HV_JSON ? (const void *)v->j + : NULL; + if (!map) { + return false; + } + for (int i = 0; i < st->npoison; i++) { + const hpoison_t *p = &st->poison[i]; + if (p->map == map) { + return true; + } + if (p->deep && p->path && v->path) { + size_t pl = strlen(p->path); + if (pl == 0 || (strncmp(v->path, p->path, pl) == 0 && + (v->path[pl] == '\0' || v->path[pl] == '.' || v->path[pl] == '['))) { + return true; + } + } + } + return false; +} + +static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth); + +/* Map v changed in place; false when v is no map (or no room to say so). A + * merge (`deep`) may change every map within it too -- a map the template + * made (dict) holds others by reference, so those are poisoned as well. */ +static bool hst_poison(hst_t *st, const hv_t *v, bool deep) { + return hst_poison_at(st, v, deep, 0); +} + +static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { + if (depth > HELM_MAX_NESTING) { + return false; + } + if (deep && v->k == HV_LIST) { + /* a list itself is never changed in place; the maps in it may be */ + for (int i = 0; i < v->l->n; i++) { + if ((hv_is_map(&v->l->items[i]) || hv_is_list(&v->l->items[i])) && + !hst_poison_at(st, &v->l->items[i], deep, depth + SKIP_ONE)) { + return false; + } + } + return true; + } + /* a values list (deep): by its path, which every item's path extends */ + const void *map = v->k == HV_MAP ? (const void *)v->m + : v->k == HV_JSON && (yyjson_mut_is_obj(v->j) || (deep && v->path)) + ? (const void *)v->j + : NULL; + if (!map) { + return false; + } + if (deep && v->k == HV_MAP) { + for (int i = 0; i < v->m->n; i++) { + if (hv_is_map(&v->m->vals[i]) && !hst_poison_at(st, &v->m->vals[i], deep, depth + 1)) { + return false; + } + } + } + if (st->npoison == st->pcap) { + int cap = st->pcap ? st->pcap * PAIR_LEN : CBM_SZ_16; + hpoison_t *g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + return false; + } + if (st->npoison) { + memcpy(g, st->poison, (size_t)st->npoison * sizeof(*g)); + } + st->poison = g; + st->pcap = cap; + } + st->poison[st->npoison++] = (hpoison_t){map, v->path, deep}; + return true; +} + static bool hn_is(TSNode n, const char *t) { return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; } @@ -1574,10 +1750,13 @@ static void hst_capture_end(hst_t *st) { memcpy(img, p->image, (size_t)p->n * sizeof(*img)); memcpy(keys, p->keys, (size_t)p->n * sizeof(*keys)); } - img[p->n] = st->cap.unk ? NULL : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); + img[p->n] = st->cap.unk || st->tainted + ? NULL + : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); /* the leaves read: a path that is the prefix of another is dropped */ const char *json = NULL; - for (int i = 0; i < st->nck; i++) { + bool whole = true; + for (int i = 0; i < st->nck && whole; i++) { size_t li = strlen(st->ckeys[i]); bool prefix = false; for (int j = 0; j < st->nck && !prefix; j++) { @@ -1587,9 +1766,13 @@ static void hst_capture_end(hst_t *st) { if (prefix || !st->ckeys[i][0]) { continue; } - char esc[CBM_SZ_1K]; + char *esc = cbm_arena_alloc(st->a, li * PAIR_LEN + SKIP_ONE); + if (!esc) { + whole = false; + break; + } size_t o = 0; - for (const char *c = st->ckeys[i]; *c && o + 3 < sizeof(esc); c++) { + for (const char *c = st->ckeys[i]; *c; c++) { if (*c == '"' || *c == '\\') { esc[o++] = '\\'; } @@ -1597,8 +1780,12 @@ static void hst_capture_end(hst_t *st) { } esc[o] = '\0'; json = cbm_arena_sprintf(st->a, "%s%s\"%s\"", json ? json : "[", json ? "," : "", esc); + whole = json != NULL; + } + keys[p->n] = json && whole ? cbm_arena_sprintf(st->a, "%s]", json) : NULL; + if (!whole || (json && !keys[p->n])) { + img[p->n] = NULL; /* a mark not written in full: no image, never one unmarked */ } - keys[p->n] = json ? cbm_arena_sprintf(st->a, "%s]", json) : NULL; p->image = img; p->keys = keys; p->n++; @@ -1720,6 +1907,9 @@ static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *int /* Field `name` of value v. */ static hv_t he_field(hst_t *st, const hv_t *v, const char *name, size_t nl) { + if (hv_poisoned(st, v)) { + return HV_UNKNOWN; + } if (v->k == HV_MAP) { for (int i = 0; i < v->m->n; i++) { if (strlen(v->m->keys[i]) == nl && memcmp(v->m->keys[i], name, nl) == 0) { @@ -1739,6 +1929,9 @@ static hv_t he_field(hst_t *st, const hv_t *v, const char *name, size_t nl) { } static hv_t he_index1(hst_t *st, const hv_t *v, const hv_t *k) { + if (hv_poisoned(st, v)) { + return HV_UNKNOWN; + } if (hv_is_map(v)) { if (k->k != HV_STR) { return HV_UNKNOWN; @@ -2074,10 +2267,413 @@ static hv_t he_strfn(hst_t *st, const char *fn, const hv_t *a, int n) { return HV_UNKNOWN; } -/* A function call `fn` with its evaluated arguments (the piped value last). */ +/* Sprig functions that change a map in place. */ +static bool he_mutator(const char *s, size_t n) { + static const char *const MUTATORS[] = { + "set", "unset", "merge", "mergeOverwrite", "mustMerge", "mustMergeOverwrite", NULL}; + return hy_in(s, n, MUTATORS); +} + +static bool he_define_may_mutate(hst_t *st, const char *name, int depth); + +/* The define a call / action names with a string literal into the arena; + * NULL (and *computed) when the name is no literal. */ +static const char *he_called_define(hst_t *st, const char *src, TSNode c, bool *computed) { + TSNode nm = {0}; + *computed = false; + if (hn_is(c, "function_call")) { + TSNode fn = hn_field(c, "function"); + TSNode args = hn_field(c, "arguments"); + uint32_t fb = ts_node_is_null(fn) ? 0 : ts_node_start_byte(fn); + size_t fl = ts_node_is_null(fn) ? 0 : ts_node_end_byte(fn) - fb; + if (fl != strlen("include") || strncmp(src + fb, "include", fl) != 0) { + return NULL; + } + nm = ts_node_is_null(args) ? (TSNode){0} : ts_node_named_child(args, 0); + } else { + nm = hn_field(c, "name"); + } + if (!hn_is(nm, "interpreted_string_literal") && !hn_is(nm, "raw_string_literal")) { + *computed = true; + return NULL; + } + uint32_t b = ts_node_start_byte(nm); + size_t l = ts_node_end_byte(nm) - b; + char *name = cbm_arena_alloc(st->a, l * 4 + SKIP_ONE); + size_t k = name ? hg_unquote(src + b, l, name) : (size_t)-1; + if (k == (size_t)-1) { + *computed = true; + return NULL; + } + name[k] = '\0'; + return name; +} + +/* How new a value is: HF_DEEP holds nothing another part holds (fromYaml, + * deepCopy, a dict / list of literals); HF_SHALLOW is a new map or list that + * may hold others' maps by reference (dict / list / omit / pick of values) -- + * a set / unset of it changes nothing else, a merge into it may. */ +typedef enum { HF_NONE = 0, HF_SHALLOW, HF_DEEP } hfresh_lvl_t; + +/* Sprig functions whose result holds nothing another part holds. */ +static bool he_own_fn(const char *s, size_t n) { + static const char *const OWN[] = { + "fromYaml", "fromJson", "fromYamlArray", "fromJsonArray", "deepCopy", "mustDeepCopy", NULL}; + return hy_in(s, n, OWN); +} + +/* Sprig functions whose result is a new map or list of what they are given. */ +static bool he_wrap_fn(const char *s, size_t n) { + static const char *const WRAP[] = {"dict", "list", "omit", "pick", NULL}; + return hy_in(s, n, WRAP); +} + +/* The text of call node c's function (a function_call's, or a bare + * identifier's); NULL when it has none. */ +static const char *he_fn_text(const char *src, TSNode c, size_t *n) { + TSNode fn = hn_is(c, "identifier") ? c : hn_field(c, "function"); + if (ts_node_is_null(fn)) { + return NULL; + } + uint32_t b = ts_node_start_byte(fn); + *n = ts_node_end_byte(fn) - b; + return src + b; +} + +/* The name text of variable node var ("" for `$`). */ +static const char *he_var_text(const char *src, TSNode var, size_t *n) { + TSNode nm = hn_field(var, "name"); + if (ts_node_is_null(nm)) { + *n = 0; + return ""; + } + uint32_t b = ts_node_start_byte(nm); + *n = ts_node_end_byte(nm) - b; + return src + b; +} + +/* The variables a scan saw made new, and how. */ +typedef struct { + const char **name; + size_t *len; + hfresh_lvl_t *lvl; + int n; + int cap; +} hfresh_t; + +static hfresh_lvl_t he_fresh_var(const hfresh_t *f, const char *src, TSNode var) { + size_t vl = 0; + const char *vs = he_var_text(src, var, &vl); + for (int i = f->n - SKIP_ONE; i >= 0; i--) { + if (f->len[i] == vl && memcmp(f->name[i], vs, vl) == 0) { + return f->lvl[i]; + } + } + return HF_NONE; +} + +static bool he_literal(TSNode x) { + return hn_is(x, "interpreted_string_literal") || hn_is(x, "raw_string_literal") || + hn_is(x, "int_literal") || hn_is(x, "float_literal") || hn_is(x, "true") || + hn_is(x, "false") || hn_is(x, "nil"); +} + +/* How new expression `val` is: a constructor's result (`... | fromYaml`, + * `dict`), or a part dug / got / indexed out of a variable that is new. */ +static hfresh_lvl_t he_fresh_value(const hfresh_t *f, const char *src, TSNode val) { + while (hn_is(val, "parenthesized_pipeline") && ts_node_named_child_count(val) == SKIP_ONE) { + val = ts_node_named_child(val, 0); + } + if (hn_is(val, "chained_pipeline")) { + uint32_t c = ts_node_named_child_count(val); + val = c ? ts_node_named_child(val, c - SKIP_ONE) : (TSNode){0}; + } + size_t fl = 0; + const char *fs = + hn_is(val, "function_call") || hn_is(val, "identifier") ? he_fn_text(src, val, &fl) : NULL; + if (!fs) { + return HF_NONE; + } + if (he_own_fn(fs, fl)) { + return HF_DEEP; + } + TSNode args = hn_field(val, "arguments"); + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + if (he_wrap_fn(fs, fl)) { + bool literal = true; /* a dict / list of literals holds no one's map */ + for (uint32_t i = 0; i < na && literal; i++) { + literal = he_literal(ts_node_named_child(args, i)); + } + return literal ? HF_DEEP : HF_SHALLOW; + } + if (na == 0) { + return HF_NONE; + } + /* a part of a deep-new value: dig takes its map last; get and index first */ + TSNode from = fl == strlen("dig") && memcmp(fs, "dig", fl) == 0 + ? ts_node_named_child(args, na - SKIP_ONE) + : (fl == strlen("get") && memcmp(fs, "get", fl) == 0) || + (fl == strlen("index") && memcmp(fs, "index", fl) == 0) + ? ts_node_named_child(args, 0) + : (TSNode){0}; + return hn_is(from, "variable") && he_fresh_var(f, src, from) == HF_DEEP ? HF_DEEP : HF_NONE; +} + +/* Is expression x deep-new: a literal, a deep-new variable or value, or a + * dict / list of such? */ +static bool he_deep_value(const hfresh_t *f, const char *src, TSNode x, int depth) { + while (hn_is(x, "parenthesized_pipeline") && ts_node_named_child_count(x) == SKIP_ONE) { + x = ts_node_named_child(x, 0); + } + if (depth > HELM_MAX_NESTING || ts_node_is_null(x)) { + return false; + } + if (he_literal(x)) { + return true; + } + if (hn_is(x, "variable")) { + return he_fresh_var(f, src, x) == HF_DEEP; + } + hfresh_lvl_t lvl = he_fresh_value(f, src, x); + if (lvl != HF_SHALLOW || !hn_is(x, "function_call")) { + return lvl == HF_DEEP; + } + size_t fl = 0; + const char *fs = he_fn_text(src, x, &fl); + if (!fs || !(fl == strlen("dict") || fl == strlen("list")) || + (memcmp(fs, "dict", fl) != 0 && memcmp(fs, "list", fl) != 0)) { + return false; /* omit / pick keep the maps they are given */ + } + TSNode args = hn_field(x, "arguments"); + for (uint32_t i = 0; !ts_node_is_null(args) && i < ts_node_named_child_count(args); i++) { + if (!he_deep_value(f, src, ts_node_named_child(args, i), depth + SKIP_ONE)) { + return false; + } + } + return true; +} + +/* Variable definition `def` (`$x := ...`) into f when what it makes is new. */ +static void he_fresh_note(hst_t *st, hfresh_t *f, const char *src, TSNode def) { + TSNode var = hn_field(def, "variable"); + TSNode val = hn_field(def, "value"); + hfresh_lvl_t lvl = ts_node_is_null(var) ? HF_NONE + : he_deep_value(f, src, val, 0) ? HF_DEEP + : he_fresh_value(f, src, val); + if (lvl == HF_NONE) { + return; + } + if (f->n == f->cap) { + int cap = f->cap ? f->cap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + size_t *gl = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gl)); + hfresh_lvl_t *gv = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gv)); + if (!g || !gl || !gv) { + return; + } + if (f->n) { + memcpy(g, f->name, (size_t)f->n * sizeof(*g)); + memcpy(gl, f->len, (size_t)f->n * sizeof(*gl)); + memcpy(gv, f->lvl, (size_t)f->n * sizeof(*gv)); + } + *f = (hfresh_t){g, gl, gv, f->n, cap}; + } + f->name[f->n] = he_var_text(src, var, &f->len[f->n]); + f->lvl[f->n] = lvl; + f->n++; +} + +/* Does expression x read `.` (a field of it)? In a with / range body `.` is + * not the frame's. */ +static bool he_reads_dot(TSNode x) { + TSTreeCursor cur = ts_tree_cursor_new(x); + bool found = false; + for (bool more = true; more && !found;) { + TSNode c = ts_tree_cursor_current_node(&cur); + found = hn_is(c, "dot") || hn_is(c, "field"); + if (found || ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return found; +} + +/* What a change of the value of expression x (in frame fr; NULL: not + * running) would reach: false when it reaches nothing another part holds -- + * x is own, or a known map, now poisoned (`deep`: a merge). A value not known + * here: true. */ +static bool he_change_reaches(hst_t *st, hframe_t *fr, TSNode x, bool dot_known, bool deep) { + if (!fr || ts_node_is_null(x) || (!dot_known && he_reads_dot(x))) { + return true; + } + /* read without recording: the part is not run, its reads are no capture's */ + int capture = st->capture; + st->capture = -SKIP_ONE; + hv_t v = he_eval(st, fr, x); + st->capture = capture; + /* a known map is poisoned even when own: what it holds is not known now */ + bool poisoned = hst_poison(st, &v, deep); + return !poisoned && !v.own; +} + +/* May running node n (of text src) change a map another part holds -- a set / + * unset / merge whose target is `.`, `$`, a field path, or a variable n did + * not make itself from a new map (`$d := dict ...`), in n or in a define it + * includes (whose changes reach only its argument)? A call's result as the + * target is a new value: no. With fr (n is a part of the running template + * not run), a target or include argument that is a value of fr is looked at: + * own, or a known map (then poisoned): no. An include of a computed name is a + * template file by path, another template's work, which no render models. + * The text a tpl renders is not looked into. */ +static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hframe_t *fr, + bool dot_known) { + if (depth > HELM_MAX_NESTING) { + return true; + } + hfresh_t fresh = {0}; /* variables n makes from a new map */ + TSTreeCursor cur = ts_tree_cursor_new(n); + bool found = false; + bool more = true; + while (more && !found && hst_step(st)) { + TSNode c = ts_tree_cursor_current_node(&cur); + const char *t = ts_node_type(c); + size_t fl = 0; + const char *fs = NULL; + if (strcmp(t, "variable_definition") == 0) { + he_fresh_note(st, &fresh, src, c); + } else if (strcmp(t, "function_call") == 0 && (fs = he_fn_text(src, c, &fl)) != NULL && + he_mutator(fs, fl)) { + TSNode args = hn_field(c, "arguments"); + TSNode target = ts_node_is_null(args) ? (TSNode){0} : ts_node_named_child(args, 0); + bool deep = fs[0] == 'm'; /* a merge reaches the maps within its target too */ + if (hn_is(target, "variable")) { + hfresh_lvl_t lvl = he_fresh_var(&fresh, src, target); + bool spared = lvl == HF_DEEP || (lvl == HF_SHALLOW && !deep); + found = !spared && he_change_reaches(st, fr, target, dot_known, deep); + } else if (ts_node_is_null(target) || hn_is(target, "dot") || hn_is(target, "field") || + hn_is(target, "selector_expression")) { + found = he_change_reaches(st, fr, target, dot_known, deep); + } + } else if (strcmp(t, "function_call") == 0 || strcmp(t, "template_action") == 0 || + strcmp(t, "block_action") == 0) { + bool computed = false; + const char *name = he_called_define(st, src, c, &computed); + if (!computed && name && he_define_may_mutate(st, name, depth + SKIP_ONE)) { + /* what the define changes is its argument (or a map within it) */ + TSNode args = hn_field(c, "arguments"); + TSNode arg = hn_is(c, "function_call") + ? (ts_node_is_null(args) ? (TSNode){0} + : ts_node_named_child(args, SKIP_ONE)) + : hn_field(c, "argument"); + found = !he_deep_value(&fresh, src, arg, 0) && + he_change_reaches(st, fr, arg, dot_known, true); + } + } else if (strcmp(t, "identifier") == 0 && (fs = he_fn_text(src, c, &fl)) != NULL && + he_mutator(fs, fl) && hn_is(ts_node_parent(c), "chained_pipeline")) { + found = true; /* `x | set`: the target is the piped value, not known here */ + } + /* a variable's name ($set) is no call */ + if (found || (strcmp(t, "variable") != 0 && ts_tree_cursor_goto_first_child(&cur))) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return found || st->stop; +} + +/* Memoized per define (a define reached again while it is scanned: no). */ +static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { + enum { HM_NO = 1, HM_YES = 2 }; + const hd_def_t *d = hd_find(st->defs, name); + if (!d) { + return false; /* a define not found is an error: nothing of it runs */ + } + if (!st->mutates) { + st->mutates = cbm_ht_create(0); + } + void *memo = st->mutates ? cbm_ht_get(st->mutates, name) : NULL; + if (memo) { + return memo == (void *)(uintptr_t)HM_YES; + } + const char *key = cbm_arena_strdup(st->a, name); + if (st->mutates && key) { + cbm_ht_set(st->mutates, key, (void *)(uintptr_t)HM_NO); + } + bool m = he_may_mutate(st, d->file->src, d->node, depth, NULL, false); + if (st->mutates && key) { + cbm_ht_set(st->mutates, key, (void *)(uintptr_t)(m ? HM_YES : HM_NO)); + } + return m; +} + +static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n); + +/* Is the result of `fn` over a own (see hv_t): a part dug / got / indexed / + * taken out of an own value; a dict / list / default / coalesce / ternary + * whose every map or list (and every value not known) is own? */ +static bool he_own_result(const char *fn, const hv_t *a, int n) { + if (n < SKIP_ONE) { + return strcmp(fn, "dict") == 0 || strcmp(fn, "list") == 0; /* empty and new */ + } + if (strcmp(fn, "dig") == 0) { + return a[n - SKIP_ONE].own; + } + if (strcmp(fn, "get") == 0 || strcmp(fn, "index") == 0 || strcmp(fn, "first") == 0 || + strcmp(fn, "last") == 0) { + return a[0].own; + } + if (strcmp(fn, "dict") == 0 || strcmp(fn, "list") == 0 || strcmp(fn, "default") == 0 || + strcmp(fn, "coalesce") == 0 || strcmp(fn, "ternary") == 0) { + for (int i = 0; i < n; i++) { + if (!a[i].own && (a[i].k == HV_UNK || hv_is_map(&a[i]) || hv_is_list(&a[i]))) { + return false; + } + } + return true; + } + return false; +} + +/* A function call `fn` with its evaluated arguments (the piped value last), + * its result marked own when it is. */ static hv_t he_apply(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { + if (he_own_fn(fn, strlen(fn))) { + return HV_UNKNOWN_OWN; /* fromYaml / deepCopy: a new value, known or not */ + } + hv_t r = he_apply_fn(st, fr, fn, a, n); + r.own = r.own || he_own_result(fn, a, n); + return r; +} + +static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { for (int i = 0; i < n; i++) { if (a[i].k == HV_UNK) { + /* an include of a known define not run: a change it would have + * made to its argument is missed, unless that one is own (one of a + * name not known -- a template file by path -- is another + * template's work, which no render models) */ + if (strcmp(fn, "include") == 0 && a[0].k == HV_STR && n == PAIR_LEN && !a[1].own) { + const char *dn = cbm_arena_strndup(st->a, a[0].s, a[0].sn); + if (!dn || he_define_may_mutate(st, dn, st->nesting)) { + st->tainted = true; + } + } + return HV_UNKNOWN; + } + if (hv_poisoned(st, &a[i])) { return HV_UNKNOWN; } } @@ -2348,16 +2944,25 @@ static hv_t he_apply(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { if (!hv_to_int(&a[0], &w) || w < 0 || !hv_strval(st, &a[1], &s)) { return HV_UNKNOWN; } + /* every line gets w spaces: a width whose output cannot fit stops here */ + size_t lines = SKIP_ONE; + for (size_t i = 0; i < s.sn; i++) { + lines += s.s[i] == '\n'; + } + if ((uint64_t)w > HELM_MAX_OUTPUT / lines) { + st->stop = true; + return HV_UNKNOWN; + } hsink_t out = {0}; if (fn[0] == 'n') { hs_append(st, &out, "\n", SKIP_ONE); } - for (int64_t i = 0; i < w; i++) { + for (int64_t i = 0; i < w && !out.unk; i++) { hs_append(st, &out, " ", SKIP_ONE); } - for (size_t i = 0; i < s.sn; i++) { + for (size_t i = 0; i < s.sn && !out.unk; i++) { hs_append(st, &out, s.s + i, SKIP_ONE); - for (int64_t k = 0; s.s[i] == '\n' && k < w; k++) { + for (int64_t k = 0; s.s[i] == '\n' && k < w && !out.unk; k++) { hs_append(st, &out, " ", SKIP_ONE); } } @@ -2423,6 +3028,30 @@ static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { fn[fl] = '\0'; TSNode args = hn_is(call, "identifier") ? (TSNode){0} : hn_field(call, "arguments"); uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + if (he_mutator(fn, fl)) { + /* set / unset / merge change a map in place -- not modelled: what it + * holds is UNKNOWN from here on (after a merge, every map within it). + * A target not known that may be one another part holds -- a variable, + * `.`, a field path; a call's result is a new value -- spoils every + * image after it. */ + TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; + hv_t target = na ? he_eval(st, fr, first) : piped ? *piped : HV_UNKNOWN; + /* what goes into it: own too, or the map now holds another's */ + bool put_own = true; + for (uint32_t i = SKIP_ONE; i < na + (piped && na ? SKIP_ONE : 0); i++) { + hv_t v = i < na ? he_eval(st, fr, ts_node_named_child(args, i)) : *piped; + put_own = put_own && (v.own || (v.k != HV_UNK && !hv_is_map(&v) && !hv_is_list(&v))); + } + bool held = ts_node_is_null(first) || hn_is(first, "variable") || hn_is(first, "dot") || + hn_is(first, "field") || hn_is(first, "selector_expression"); + /* a known map is poisoned even when own: what it holds is not known now */ + bool poisoned = hst_poison(st, &target, fn[0] == 'm'); + if (!poisoned && !target.own && (held || hv_is_map(&target))) { + st->tainted = true; + } + /* the changed map is what set / merge return */ + return target.own && put_own ? HV_UNKNOWN_OWN : HV_UNKNOWN; + } bool lazy = strcmp(fn, "and") == 0 || strcmp(fn, "or") == 0; if (lazy) { hv_t v = HV_UNKNOWN; @@ -2485,7 +3114,7 @@ static hv_t he_literal_int(hframe_t *fr, TSNode n) { } static hv_t he_eval(hst_t *st, hframe_t *fr, TSNode n) { - if (ts_node_is_null(n) || ts_node_has_error(n) || ts_node_is_missing(n)) { + if (ts_node_is_null(n) || ts_node_has_error(n) || ts_node_is_missing(n) || !hst_step(st)) { return HV_UNKNOWN; } const char *t = ts_node_type(n); @@ -2593,7 +3222,7 @@ static void he_item(hst_t *st, hframe_t *fr, TSNode c); /* The named children of `parent` in field `field` (NULL: every one). */ static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field) { uint32_t cc = ts_node_child_count(parent); - for (uint32_t i = 0; i < cc && !st->brk && !st->cont; i++) { + for (uint32_t i = 0; i < cc && !st->brk && !st->cont && hst_step(st); i++) { TSNode c = ts_node_child(parent, i); if (!ts_node_is_named(c)) { continue; @@ -2606,6 +3235,140 @@ static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field) } } +/* Is expression x deep-new (f: what the scan saw made new), or a variable of + * frame fr whose value is own? */ +static bool he_part_own(hst_t *st, hframe_t *fr, const hfresh_t *f, TSNode x) { + if (he_deep_value(f, fr->src, x, 0)) { + return true; + } + if (!hn_is(x, "variable")) { + return false; + } + size_t l = 0; + const char *name = hn_var_name(fr, x, &l); + hvar_t *v = hst_var(st, fr, name, l); + return v && v->v.own; +} + +/* Does assigning expression `value` to variable var keep it own (when it is): + * a deep-new value, or a set / merge of var itself with own values + * (`$dst = ... | fromYaml | merge $dst`)? */ +static bool he_assign_owns(hst_t *st, hframe_t *fr, const hfresh_t *f, TSNode var, TSNode value) { + const char *src = fr->src; + if (he_deep_value(f, src, value, 0)) { + return true; + } + while (hn_is(value, "parenthesized_pipeline") && ts_node_named_child_count(value) == SKIP_ONE) { + value = ts_node_named_child(value, 0); + } + TSNode call = value; + TSNode piped = {0}; + if (hn_is(value, "chained_pipeline")) { + uint32_t c = ts_node_named_child_count(value); + if (c < PAIR_LEN) { + return false; + } + call = ts_node_named_child(value, c - SKIP_ONE); + piped = ts_node_named_child(value, c - PAIR_LEN); + } + size_t fl = 0; + const char *fs = hn_is(call, "function_call") ? he_fn_text(src, call, &fl) : NULL; + TSNode args = fs && he_mutator(fs, fl) ? hn_field(call, "arguments") : (TSNode){0}; + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; + if (!hn_is(first, "variable")) { + return false; + } + size_t al = 0; + size_t bl = 0; + const char *as = he_var_text(src, first, &al); + const char *bs = he_var_text(src, var, &bl); + if (al != bl || memcmp(as, bs, al) != 0) { + return false; + } + for (uint32_t i = SKIP_ONE; i < na; i++) { + if (!he_part_own(st, fr, f, ts_node_named_child(args, i))) { + return false; + } + } + return ts_node_is_null(piped) || he_part_own(st, fr, f, piped); +} + +/* The variables of the frames around n that n assigns (`$x = ...`) become + * UNKNOWN (an own one stays own when what n gives it keeps it so); true when + * n would leave the loop around it -- a break / continue not inside a range + * of n's own (`in_range`: n is a range's body). */ +static bool he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { + TSTreeCursor cur = ts_tree_cursor_new(n); + hfresh_t fresh = {0}; /* variables n makes new */ + bool leave = false; + bool more = true; + while (more && hst_step(st)) { + TSNode c = ts_tree_cursor_current_node(&cur); + const char *t = ts_node_type(c); + if (strcmp(t, "variable_definition") == 0) { + he_fresh_note(st, &fresh, fr->src, c); + } else if (strcmp(t, "assignment") == 0) { + TSNode var = hn_field(c, "variable"); + size_t l = 0; + const char *name = ts_node_is_null(var) ? NULL : hn_var_name(fr, var, &l); + hvar_t *v = name ? hst_var(st, fr, name, l) : NULL; + if (v) { + bool keeps = v->v.own && he_assign_owns(st, fr, &fresh, var, hn_field(c, "value")); + v->v = keeps ? HV_UNKNOWN_OWN : HV_UNKNOWN; + } + } else if (strcmp(t, "break_action") == 0 || strcmp(t, "continue_action") == 0) { + /* a range around it inside n (n itself included) is the one it leaves */ + bool own = in_range || hn_is(n, "range_action"); + for (TSNode p = c; !own && !ts_node_eq(p, n) && !ts_node_is_null(p);) { + p = ts_node_parent(p); + own = hn_is(p, "range_action"); + } + leave = leave || !own; + } + if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return leave; +} + +/* The bodies of if / with / range n, not run because its condition or + * collection is UNKNOWN (that one was run): their output, the variables they + * would assign and a change they would make are UNKNOWN; a loop they would + * leave is left now (its later iterations are not known). */ +static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { + hst_out_unknown(st, fr); + bool range = hn_is(n, "range_action"); + bool with = hn_is(n, "with_action"); + bool leave = false; + bool mutates = false; + uint32_t cc = ts_node_child_count(n); + for (uint32_t i = 0; i < cc; i++) { + TSNode c = ts_node_child(n, i); + const char *f = ts_node_field_name_for_child(n, i); + if (!ts_node_is_named(c) || hn_is(c, "range_variable_definition") || + (f && (strcmp(f, "condition") == 0 || strcmp(f, "range") == 0))) { + continue; + } + leave = he_scan_skipped(st, fr, c, range) || leave; + mutates = mutates || he_may_mutate(st, fr->src, c, st->nesting, fr, !range && !with); + } + if (mutates) { + st->tainted = true; + } + if (leave) { + st->brk = true; + } +} + /* if / else if / else, and with / else with: the first branch whose * condition holds runs; an UNKNOWN condition runs none. */ static void he_branches(hst_t *st, hframe_t *fr, TSNode n, bool with) { @@ -2630,10 +3393,10 @@ static void he_branches(hst_t *st, hframe_t *fr, TSNode n, bool with) { continue; } hv_t v = he_eval(st, fr, c); - if (v.k == HV_UNK) { + if (v.k == HV_UNK || hv_poisoned(st, &v)) { stop = true; active = false; - hst_out_unknown(st, fr); /* which branch renders is unknown */ + he_skipped(st, fr, n); /* which branch renders is unknown */ continue; } active = hv_truth(&v); @@ -2670,18 +3433,18 @@ static void he_range(hst_t *st, hframe_t *fr, TSNode n) { ivar = (TSNode){0}; } hv_t coll = he_eval(st, fr, expr); - if (coll.k == HV_UNK) { - hst_out_unknown(st, fr); + if (coll.k == HV_UNK || hv_poisoned(st, &coll)) { + he_skipped(st, fr, n); return; } hv_t *items = NULL; const char **keys = NULL; - int count = 0; + int64_t count = 0; bool is_map = hv_is_map(&coll); if (is_map) { count = hv_keys(st, &coll, &keys); } else if (coll.k == HV_INT) { - count = coll.i > 0 ? (int)coll.i : 0; + count = coll.i > 0 ? coll.i : 0; } else { count = hv_items(st, &coll, &items); } @@ -2693,7 +3456,7 @@ static void he_range(hst_t *st, hframe_t *fr, TSNode n) { size_t el = 0; const char *iname = ts_node_is_null(ivar) ? NULL : hn_var_name(fr, ivar, &il); const char *ename = ts_node_is_null(evar) ? NULL : hn_var_name(fr, evar, &el); - for (int k = 0; k < count; k++) { + for (int64_t k = 0; k < count && hst_step(st); k++) { int mark = st->nvars; hv_t elem; hv_t index; @@ -2742,8 +3505,13 @@ static void he_template(hst_t *st, hframe_t *fr, TSNode n) { } name[k] = '\0'; hv_t a = ts_node_is_null(arg) ? hv_nil() : he_eval(st, fr, arg); - if (a.k == HV_UNK) { + if (a.k == HV_UNK || hv_poisoned(st, &a)) { hst_out_unknown(st, fr); + /* the define not run: a change it would make to its argument is + * missed, unless that one is own */ + if (!a.own && he_define_may_mutate(st, name, st->nesting)) { + st->tainted = true; + } return; } if (fr->out) { @@ -2837,17 +3605,16 @@ static hv_t hv_cstr(const char *s) { return s ? hv_str(s, strlen(s)) : hv_nil(); } -void cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, +bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes) { - (void)rel; for (int i = 0; i < nprobes; i++) { probes[i].n = 0; probes[i].image = NULL; probes[i].keys = NULL; } if (!src || nprobes == 0) { - return; + return true; } hst_t st = {.a = a, .defs = defs, @@ -2866,7 +3633,7 @@ void cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, if (st.scratch) { yyjson_mut_doc_free(st.scratch); } - return; + return true; } int nl = SKIP_ONE; for (size_t i = 0; i < len; i++) { @@ -2909,4 +3676,20 @@ void cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, } ts_parser_delete(st.parser); yyjson_mut_doc_free(st.scratch); + if (st.mutates) { + cbm_ht_free(st.mutates); + } + if (st.stop) { + /* Helm would not finish this template: none of its images is known, + * also those captured before the budget ran out */ + for (int i = 0; i < nprobes; i++) { + for (int c = 0; c < probes[i].n; c++) { + probes[i].image[c] = NULL; + probes[i].keys[c] = NULL; + } + } + cbm_log_warn("helm.render.budget", "file", rel ? rel : "", "reason", + st.steps > HELM_MAX_STEPS ? "steps" : "output"); + } + return !st.stop; } diff --git a/src/pipeline/helm_template.h b/src/pipeline/helm_template.h index ad24a6f78..47bdb007b 100644 --- a/src/pipeline/helm_template.h +++ b/src/pipeline/helm_template.h @@ -75,8 +75,10 @@ typedef struct { /* Render template `rel` (text src, len) with `values` (its .Values; NULL: * an empty map), `chart` and the family's defines, filling `probes` (strings * in arena a). Values paths are dot-joined keys, a key's own `.` escaped - * (`\.`), as --set writes them. */ -void cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, + * (`\.`), as --set writes them. False when the render ran past its step or + * output budget -- a template Helm would not finish either (logged): then + * every capture is UNKNOWN. */ +bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes); diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c index a44a72c9c..486b1a5b2 100644 --- a/src/pipeline/pass_ci.c +++ b/src/pipeline/pass_ci.c @@ -1086,7 +1086,8 @@ static int ci_compose_env(const ci_t *c, TSNode svc, const ci_env_t *dotenv, con /* A compose build's `args` (a mapping, or a list of "K=V" items), their * values interpolated, as a JSON object; NULL: none. An argument without a - * value (the environment's) or with an unknown one is left out. */ + * value (the environment's), with an unknown one or with a secret (by name or + * value -- never kept in the graph, never an image) is left out. */ static const char *ci_compose_build_args(const ci_t *c, TSNode build, const ci_env_t *env) { TSNode args = ts_node_is_null(build) ? build : ci_get(c, build, "args"); if (ts_node_is_null(args)) { @@ -1109,13 +1110,19 @@ static const char *ci_compose_build_args(const ci_t *c, TSNode build, const ci_e raw = ts_node_is_null(pairs[i].value) ? NULL : ci_scalar(c, pairs[i].value, NULL); } const char *value = raw ? ci_interp(c->a, raw, env) : NULL; - if (!key || !key[0] || !value) { + if (!key || !key[0] || !value || cbm_is_secret_binding(key, value)) { continue; } - char ek[CBM_SZ_256]; - char ev[CBM_SZ_1K]; - cbm_json_escape(ek, (int)sizeof(ek), key); - cbm_json_escape(ev, (int)sizeof(ev), value); + /* JSON escaping writes at most 6 bytes a byte: nothing is cut */ + int kn = (int)(strlen(key) * 6 + SKIP_ONE); + int vn = (int)(strlen(value) * 6 + SKIP_ONE); + char *ek = cbm_arena_alloc(c->a, (size_t)kn); + char *ev = cbm_arena_alloc(c->a, (size_t)vn); + if (!ek || !ev) { + return NULL; + } + cbm_json_escape(ek, kn, key); + cbm_json_escape(ev, vn, value); json = cbm_arena_sprintf(c->a, "%s%s\"%s\":\"%s\"", json ? json : "{", json ? "," : "", ek, ev); } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 60433514c..9a945c1d5 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -3939,22 +3939,45 @@ static void k8s_helm_collect(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f, k8s_ } } +/* Repository file `rel`, read only when discovery took it in: a values file, + * a Chart.yaml, a kustomize valuesFile named by path is never a symlink or + * under one -- nothing is read from outside the repository. */ static char *k8s_repo_read(cbm_pipeline_ctx_t *ctx, const char *rel, int *len) { + if (!cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, rel)) { + return NULL; + } char path[CBM_SZ_4K]; - snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); - return k8s_read_file(path, len); + int n = snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + return n > 0 && (size_t)n < sizeof(path) ? k8s_read_file(path, len) : NULL; } -/* A file of `dir` (one of `names`), parsed as values; *found when one exists. */ +/* Is repository path `rel` there at all -- a file, a symlink, anything (a + * link is reported, never followed)? */ +static bool k8s_repo_has(cbm_pipeline_ctx_t *ctx, const char *rel) { + char path[CBM_SZ_4K]; + int n = snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + cbm_path_info_t info; + return n > 0 && (size_t)n < sizeof(path) && cbm_path_info_utf8(path, &info) == 0; +} + +/* A file of `dir` (one of `names`), parsed as values; *found when one exists. + * *unread when one is there but is not read (a symlink, too large): what it + * holds is unknown, which is not the same as no file. */ static yyjson_mut_val *k8s_helm_yaml(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const char *dir, - const char *const *names, bool *found) { + const char *const *names, bool *found, bool *unread) { *found = false; + *unread = false; for (int i = 0; names[i]; i++) { char rel[CBM_SZ_1K]; snprintf(rel, sizeof(rel), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); int len = 0; char *src = k8s_repo_read(ctx, rel, &len); if (!src) { + if (k8s_repo_has(ctx, rel)) { + *found = true; + *unread = true; + return NULL; + } continue; } *found = true; @@ -3986,9 +4009,13 @@ static k8s_hchart_t *k8s_helm_chart(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_ static const char *const CHART[] = {"Chart.yaml", "Chart.yml", NULL}; static const char *const VALUES[] = {"values.yaml", "values.yml", NULL}; bool found = false; - c->chart_yaml = k8s_helm_yaml(ctx, h, dir, CHART, &found); - yyjson_mut_val *vals = k8s_helm_yaml(ctx, h, dir, VALUES, &found); - c->values = found ? vals : yyjson_mut_obj(h->doc); + bool chart_unread = false; + bool values_unread = false; + c->chart_yaml = k8s_helm_yaml(ctx, h, dir, CHART, &found, &chart_unread); + yyjson_mut_val *vals = k8s_helm_yaml(ctx, h, dir, VALUES, &found, &values_unread); + /* a Chart.yaml or values file there but not read: the chart's values are + * unknown (it renders nothing), never empty */ + c->values = chart_unread || values_unread ? NULL : found ? vals : yyjson_mut_obj(h->doc); const cbm_gbuf_node_t *cn = c->chart > 0 ? cbm_gbuf_find_by_id(ctx->gbuf, c->chart) : NULL; const char *name = k8s_helm_meta_str(c->chart_yaml, "name"); c->meta.name = name ? name : cn && cn->name ? cbm_arena_strdup(&h->a, cn->name) : NULL; @@ -4139,12 +4166,12 @@ static yyjson_mut_val *k8s_helm_dig(yyjson_mut_val *v, const char *path) { } /* Is values key path `key` (dot-joined, `\.` escaped) set in `v`? */ -static bool k8s_helm_sets(yyjson_mut_val *v, const char *key) { - char part[CBM_SZ_512]; +static bool k8s_helm_sets(k8s_helm_t *h, yyjson_mut_val *v, const char *key) { + char *part = cbm_arena_alloc(&h->a, strlen(key) + SKIP_ONE); /* a key is never cut */ const char *p = key; - while (v && *p) { + while (v && part && *p) { size_t k = 0; - while (*p && *p != '.' && *p != '[' && k + SKIP_ONE < sizeof(part)) { + while (*p && *p != '.' && *p != '[') { if (*p == '\\' && p[1]) { p++; } @@ -4180,13 +4207,19 @@ static const char *k8s_helm_overridden(k8s_helm_t *h, const char *keys, yyjson_m yyjson_arr_foreach(arr, idx, max, k) { const char *key = yyjson_get_str(k); bool global = key && strncmp(key, "global.", strlen("global.")) == 0; - bool set = key && (global ? k8s_helm_sets(globals, key + strlen("global.")) - : k8s_helm_sets(overlay, key)); + bool set = key && (global ? k8s_helm_sets(h, globals, key + strlen("global.")) + : k8s_helm_sets(h, overlay, key)); if (!set) { continue; } - char esc[CBM_SZ_1K]; - cbm_json_escape(esc, (int)sizeof(esc), key); + /* JSON escaping writes at most 6 bytes a byte */ + int en = (int)(strlen(key) * 6 + SKIP_ONE); + char *esc = cbm_arena_alloc(&h->a, (size_t)en); + if (!esc) { + out = NULL; + break; + } + cbm_json_escape(esc, en, key); out = cbm_arena_sprintf(&h->a, "%s%s\"%s%s%s\"", out ? out : "[", out ? "," : "", global || !prefix ? "" : prefix, global || !prefix ? "" : ".", esc); @@ -4245,14 +4278,18 @@ static void k8s_helm_take(cbm_pipeline_ctx_t *ctx, const cbm_helm_probe_t *p, in /* The images of templated containers [from, to) of one template, rendered * with `values` (NULL: the chart's own). */ static void k8s_helm_render(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, - const k8s_hchart_t *c, const k8s_timg_t *t, int from, int to, - yyjson_mut_val *values, const char *src, size_t len, + const k8s_hchart_t *c, const char *as_name, const k8s_timg_t *t, + int from, int to, yyjson_mut_val *values, const char *src, size_t len, cbm_helm_probe_t *probes, int np) { char fam[CBM_SZ_1K]; k8s_chart_family(b, c->dir, fam, sizeof(fam)); const cbm_helm_defines_t *defs = k8s_helm_defines(ctx, b, h, fam); - cbm_helm_render(&h->a, t[from].file, src, len, values ? values : c->values, &c->meta, defs, - probes, np); + /* a subchart a dependency aliases is .Chart.Name = its alias (Helm's + * getAliasDependency) */ + cbm_helm_chart_meta_t meta = c->meta; + meta.name = as_name ? as_name : meta.name; + cbm_helm_render(&h->a, t[from].file, src, len, values ? values : c->values, &meta, defs, probes, + np); (void)to; } @@ -4381,11 +4418,13 @@ static bool k8s_helm_disabled(k8s_helm_t *h, yyjson_mut_val *dep, yyjson_mut_val * with values `top_vals` (NULL: top's defaults): down the charts between * them, each subchart's defaults with its parent's `:` values and * `global:` coalesced over them (Helm's coalesce; a subchart always has a - * `global`). *path gets dir's key path from top ("" for top itself). NULL: - * a dependency condition disables it, or a values file does not parse. */ + * `global`). *path gets dir's key path from top ("" for top itself), *as_name + * the name dir's chart is included under (its alias; NULL for top). NULL: a + * dependency condition disables it, or a values file does not parse. */ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, k8s_dep_link_t *l, const char *dir, const char *top, - yyjson_mut_val *top_vals, const char **path) { + yyjson_mut_val *top_vals, const char **path, + const char **as_name) { const char **anc = NULL; int na = k8s_helm_ancestors(h, b, dir, &anc); int ti = -SKIP_ONE; /* top's index among the ancestors; na: dir itself */ @@ -4397,6 +4436,7 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, } } *path = ""; + *as_name = NULL; if (ti < 0) { return NULL; } @@ -4422,6 +4462,7 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, } *path = (*path)[0] ? cbm_arena_sprintf(&h->a, "%s.%s", *path, alias) : cbm_arena_strdup(&h->a, alias); + *as_name = alias; yyjson_mut_val *sub = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, alias) : NULL; yyjson_mut_val *next = cbm_helm_values_coalesce(h->doc, child->values, sub); yyjson_mut_val *pg = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, "global") : NULL; @@ -4441,10 +4482,10 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, /* Render the templated containers [from, to) of one template with `vals` * and take each container's images. */ static void k8s_helm_images_of(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, - const k8s_hchart_t *c, int from, int to, yyjson_mut_val *vals, - const char *src, size_t len, cbm_helm_probe_t *probes, int np, - k8s_himgs_t *out) { - k8s_helm_render(ctx, h, b, c, h->t, from, to, vals, src, len, probes, np); + const k8s_hchart_t *c, const char *as_name, int from, int to, + yyjson_mut_val *vals, const char *src, size_t len, + cbm_helm_probe_t *probes, int np, k8s_himgs_t *out) { + k8s_helm_render(ctx, h, b, c, as_name, h->t, from, to, vals, src, len, probes, np); for (int i = from; i < to; i++) { k8s_helm_take(ctx, probes, np, h->t[i].start_line, h->t[i].end_line, h->t[i].node, h->t[i].container, out); @@ -4471,17 +4512,19 @@ static void k8s_helm_link_defaults(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 ? k8s_helm_chart(ctx, h, &l, dir) : NULL; const char *path = NULL; + const char *as_name = NULL; yyjson_mut_val *vals = NULL; if (c) { k8s_chart_family(b, c->dir, fam, sizeof(fam)); - vals = k8s_helm_context(ctx, h, b, &l, c->dir, fam, NULL, &path); + vals = k8s_helm_context(ctx, h, b, &l, c->dir, fam, NULL, &path, &as_name); } int len = 0; char *src = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; cbm_helm_probe_t *probes = NULL; int np = src ? k8s_helm_probes(h, h->t, from, to, src, (size_t)len, &probes) : 0; if (np > 0) { - k8s_helm_images_of(ctx, h, b, c, from, to, vals, src, (size_t)len, probes, np, &uses); + k8s_helm_images_of(ctx, h, b, c, as_name, from, to, vals, src, (size_t)len, probes, np, + &uses); } cbm_free(CBM_MEM_CLASS_OTHER, src); from = to; @@ -4535,14 +4578,17 @@ static void k8s_helm_release_local(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 (strcmp(dir, top) == 0 || (tl == 0 || (strncmp(dir, top, tl) == 0 && dir[tl] == '/'))); k8s_hchart_t *c = in ? k8s_helm_chart(ctx, h, l, dir) : NULL; const char *path = NULL; - yyjson_mut_val *vals = c ? k8s_helm_context(ctx, h, b, l, dir, top, rv, &path) : NULL; + const char *as_name = NULL; + yyjson_mut_val *vals = + c ? k8s_helm_context(ctx, h, b, l, dir, top, rv, &path, &as_name) : NULL; int len = 0; char *srcs = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; cbm_helm_probe_t *probes = NULL; int np = srcs ? k8s_helm_probes(h, h->t, from, to, srcs, (size_t)len, &probes) : 0; if (np > 0) { k8s_himgs_t got = {0}; - k8s_helm_images_of(ctx, h, b, c, from, to, vals, srcs, (size_t)len, probes, np, &got); + k8s_helm_images_of(ctx, h, b, c, as_name, from, to, vals, srcs, (size_t)len, probes, np, + &got); yyjson_mut_val *overlay = path && path[0] ? k8s_helm_dig(rv, path) : rv; yyjson_mut_val *globals = yyjson_mut_obj_get(rv, "global"); k8s_helm_emit_overrides(ctx, h, src, via, &h->dflt, &got, overlay, globals, @@ -4700,8 +4746,40 @@ static bool k8s_json_str(const char *props, const char *key, char *out, size_t s return ok; } +/* Does field `key` of `obj` give a value (a non-null scalar, a non-empty list + * or map)? */ +static bool k8s_helm_gives(yyjson_mut_val *obj, const char *key) { + yyjson_mut_val *v = yyjson_mut_is_obj(obj) ? yyjson_mut_obj_get(obj, key) : NULL; + if (!v || yyjson_mut_is_null(v)) { + return false; + } + if (yyjson_mut_is_arr(v)) { + return yyjson_mut_arr_size(v) > 0; + } + if (yyjson_mut_is_obj(v)) { + return yyjson_mut_obj_size(v) > 0; + } + return !yyjson_mut_is_str(v) || yyjson_mut_get_len(v) > 0; +} + +/* A HelmRelease's values: spec.values; NULL when there are none -- or when a + * source not read also gives some (valuesFrom, the chart's valuesFiles, a + * postRenderer that may patch the images): its images are then unknown. */ +static yyjson_mut_val *k8s_helm_flux_values(yyjson_mut_val *docv) { + yyjson_mut_val *spec = yyjson_mut_is_obj(docv) ? yyjson_mut_obj_get(docv, "spec") : NULL; + yyjson_mut_val *chart = yyjson_mut_is_obj(spec) ? yyjson_mut_obj_get(spec, "chart") : NULL; + yyjson_mut_val *cspec = yyjson_mut_is_obj(chart) ? yyjson_mut_obj_get(chart, "spec") : NULL; + if (k8s_helm_gives(spec, "valuesFrom") || k8s_helm_gives(spec, "postRenderers") || + k8s_helm_gives(cspec, "valuesFiles") || k8s_helm_gives(cspec, "valuesFile")) { + return NULL; + } + return yyjson_mut_is_obj(spec) ? yyjson_mut_obj_get(spec, "values") : NULL; +} + /* A kustomize helmCharts entry's values: valuesFile, then valuesInline over - * it; NULL when neither. */ + * it; NULL when neither -- or when a part is not read (additionalValuesFiles, + * a valuesMerge other than the default override, a valuesFile that cannot be + * read): its images are then unknown. */ static yyjson_mut_val *k8s_helm_kustomize_values(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const cbm_gbuf_node_t *kust, const char *props) { char chart[CBM_SZ_512] = ""; @@ -4723,6 +4801,11 @@ static yyjson_mut_val *k8s_helm_kustomize_values(cbm_pipeline_ctx_t *ctx, k8s_he const char *rel = k8s_helm_meta_str(c, "releaseName"); if (name && strcmp(name, chart) == 0 && (!release[0] || (rel && strcmp(rel, release) == 0))) { + const char *merge = k8s_helm_meta_str(c, "valuesMerge"); + if (k8s_helm_gives(c, "additionalValuesFiles") || + (merge && strcmp(merge, "override") != 0)) { + return NULL; + } inline_vals = yyjson_mut_obj_get(c, "valuesInline"); break; } @@ -4733,6 +4816,9 @@ static yyjson_mut_val *k8s_helm_kustomize_values(cbm_pipeline_ctx_t *ctx, k8s_he char *fs = k8s_repo_read(ctx, vfile, &fl); file_vals = fs ? cbm_helm_values_parse(h->doc, fs, (size_t)fl) : NULL; cbm_free(CBM_MEM_CLASS_OTHER, fs); + if (!file_vals) { + return NULL; /* a valuesFile not read: the release's values would be partial */ + } } if (!inline_vals && !file_vals) { return NULL; @@ -4772,10 +4858,7 @@ static void k8s_helm_link_releases(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 bool hr = strstr(rels[i].props, "\"via\":\"helmrelease\"") != NULL; yyjson_mut_val *rv = NULL; if (hr) { - yyjson_mut_val *docv = k8s_helm_node_yaml(ctx, h, src); - yyjson_mut_val *spec = - yyjson_mut_is_obj(docv) ? yyjson_mut_obj_get(docv, "spec") : NULL; - rv = yyjson_mut_is_obj(spec) ? yyjson_mut_obj_get(spec, "values") : NULL; + rv = k8s_helm_flux_values(k8s_helm_node_yaml(ctx, h, src)); } else { rv = k8s_helm_kustomize_values(ctx, h, src, rels[i].props); } @@ -4840,14 +4923,13 @@ typedef struct { int64_t chart; int64_t img; bool literal; + bool lost; /* a mark that could not be carried whole: no edge, never an unmarked one */ char *keys; /* JSON array (cbm_alloc); NULL: none yet */ } k8s_cimg_t; -/* JSON array `a` (owned, may be NULL) with the strings of JSON array `b` - * appended once; the result is owned. */ -static char *k8s_keys_union(char *a, const char *b) { - yyjson_doc *db = b ? yyjson_read(b, strlen(b), 0) : NULL; - yyjson_val *bv = db ? yyjson_doc_get_root(db) : NULL; +/* JSON array `a` (owned, may be NULL) with the strings of JSON array value + * `bv` appended once; the result is owned. *lost when one could not be. */ +static char *k8s_keys_union(char *a, yyjson_val *bv, bool *lost) { size_t idx, max; yyjson_val *k; yyjson_arr_foreach(bv, idx, max, k) { @@ -4855,29 +4937,38 @@ static char *k8s_keys_union(char *a, const char *b) { if (!s) { continue; } - char esc[CBM_SZ_1K]; - char quoted[CBM_SZ_1K + PAIR_LEN + PAIR_LEN]; - cbm_json_escape(esc, (int)sizeof(esc), s); - snprintf(quoted, sizeof(quoted), "\"%s\"", esc); + /* JSON escaping writes at most 6 bytes (\u00XX) a byte */ + size_t qn = strlen(s) * 6 + PAIR_LEN + SKIP_ONE; + char *quoted = cbm_alloc(CBM_MEM_CLASS_OTHER, qn); + if (!quoted) { + *lost = true; + break; + } + quoted[0] = '"'; + cbm_json_escape(quoted + SKIP_ONE, (int)(qn - PAIR_LEN), s); + size_t ql = strlen(quoted); + quoted[ql++] = '"'; + quoted[ql] = '\0'; if (a && strstr(a, quoted)) { + cbm_free(CBM_MEM_CLASS_OTHER, quoted); continue; } size_t al = a ? strlen(a) : 0; - char *g = cbm_alloc(CBM_MEM_CLASS_OTHER, al + strlen(quoted) + 3); + char *g = cbm_alloc(CBM_MEM_CLASS_OTHER, al + ql + 3); if (!g) { + cbm_free(CBM_MEM_CLASS_OTHER, quoted); + *lost = true; break; } if (a) { - snprintf(g, al + strlen(quoted) + 3, "%.*s,%s]", (int)(al - SKIP_ONE), a, quoted); + snprintf(g, al + ql + 3, "%.*s,%s]", (int)(al - SKIP_ONE), a, quoted); } else { - snprintf(g, strlen(quoted) + 3, "[%s]", quoted); + snprintf(g, ql + 3, "[%s]", quoted); } + cbm_free(CBM_MEM_CLASS_OTHER, quoted); cbm_free(CBM_MEM_CLASS_OTHER, a); a = g; } - if (db) { - yyjson_doc_free(db); - } return a; } @@ -4923,30 +5014,41 @@ static void k8s_link_chart_images(cbm_pipeline_ctx_t *ctx) { if (!k8s_grow((void **)&v, &cap, n, sizeof(*v))) { break; } - v[n] = (k8s_cimg_t){charts[c]->id, img->id, false, NULL}; + v[n] = (k8s_cimg_t){charts[c]->id, img->id, false, false, NULL}; at = n++; } - const char *dflt = strstr(up, "\"default_of\":"); - if (!dflt) { + /* the USAGE's default_of, read as JSON (a key path may hold `]`) */ + yyjson_doc *ud = yyjson_read(up, strlen(up), 0); + yyjson_val *dv = ud ? yyjson_obj_get(yyjson_doc_get_root(ud), "default_of") : NULL; + if (!ud) { + v[at].lost = true; + } else if (!dv) { v[at].literal = true; + } else if (yyjson_is_arr(dv)) { + v[at].keys = k8s_keys_union(v[at].keys, dv, &v[at].lost); } else { - char keys[CBM_SZ_4K]; - const char *arr = dflt + strlen("\"default_of\":"); - const char *end = strchr(arr, ']'); - if (end && (size_t)(end - arr) + PAIR_LEN < sizeof(keys)) { - snprintf(keys, sizeof(keys), "%.*s", (int)(end - arr + SKIP_ONE), arr); - v[at].keys = k8s_keys_union(v[at].keys, keys); - } + v[at].lost = true; + } + if (ud) { + yyjson_doc_free(ud); } } } } for (int i = 0; i < n; i++) { - char props[CBM_SZ_4K]; bool marked = !v[i].literal && v[i].keys; - snprintf(props, sizeof(props), "{\"via\":\"chart\",\"ref\":\"image\"%s%s}", - marked ? ",\"default_of\":" : "", marked ? v[i].keys : ""); - cbm_gbuf_insert_edge(ctx->gbuf, v[i].chart, v[i].img, "DEPLOYS", props); + if (!v[i].literal && v[i].lost) { + cbm_free(CBM_MEM_CLASS_OTHER, v[i].keys); + continue; /* the mark is not whole: no edge rather than an unmarked one */ + } + size_t pn = (marked ? strlen(v[i].keys) : 0) + CBM_SZ_64; + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, pn); + if (props) { + snprintf(props, pn, "{\"via\":\"chart\",\"ref\":\"image\"%s%s}", + marked ? ",\"default_of\":" : "", marked ? v[i].keys : ""); + cbm_gbuf_insert_edge(ctx->gbuf, v[i].chart, v[i].img, "DEPLOYS", props); + cbm_free(CBM_MEM_CLASS_OTHER, props); + } cbm_free(CBM_MEM_CLASS_OTHER, v[i].keys); } cbm_free(CBM_MEM_CLASS_OTHER, v); diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index e3a1e768a..334322d15 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -2489,7 +2489,8 @@ static const char *sh_image_repo(sh_t *s, TSNode cmd, const sh_argv_t *a, int i) } /* JSON object `json` (NULL: none) with build argument `kv` ("K=V") added; a - * `K` alone (the environment's value) or an unknown word adds nothing. */ + * `K` alone (the environment's value), an unknown word or a secret (by name + * or value -- never kept in the graph, never an image) adds nothing. */ static const char *sh_build_arg(sh_t *s, const char *json, const char *kv) { const char *eq = kv ? strchr(kv, '=') : NULL; if (!eq || eq == kv) { @@ -2500,8 +2501,17 @@ static const char *sh_build_arg(sh_t *s, const char *json, const char *kv) { return json; } } - char ev[CBM_SZ_1K]; - cbm_json_escape(ev, (int)sizeof(ev), eq + SKIP_ONE); + const char *key = cbm_arena_strndup(s->a, kv, (size_t)(eq - kv)); + if (!key || cbm_is_secret_binding(key, eq + SKIP_ONE)) { + return json; + } + /* JSON escaping writes at most 6 bytes a byte: nothing is cut */ + int vn = (int)(strlen(eq + SKIP_ONE) * 6 + SKIP_ONE); + char *ev = cbm_arena_alloc(s->a, (size_t)vn); + if (!ev) { + return json; + } + cbm_json_escape(ev, vn, eq + SKIP_ONE); const char *pair = cbm_arena_sprintf(s->a, "\"%.*s\":\"%s\"", (int)(eq - kv), kv, ev); if (!pair) { return json; @@ -3763,17 +3773,25 @@ static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTabl char esc_file[CBM_SZ_1K]; char esc_img[CBM_SZ_1K]; char ci[CBM_SZ_1K]; - char props[CBM_SZ_4K]; cbm_json_escape(esc_file, (int)sizeof(esc_file), x->file); cbm_json_escape(esc_img, (int)sizeof(esc_img), x->ext); sh_ci_props(x, ci, sizeof(ci)); - snprintf(props, sizeof(props), + /* sized to what it holds: default_of and build_args are never cut */ + size_t pn = strlen(esc_file) + strlen(esc_img) + strlen(ci) + + (x->default_of ? strlen(x->default_of) : 0) + + (x->build_args ? strlen(x->build_args) : 0) + CBM_SZ_256; + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, pn); + if (!props) { + return 0; + } + snprintf(props, pn, "{\"via\":\"build\",\"tool\":\"docker-build\",\"file\":\"%s\",\"image\":\"%s\"%s," "\"line\":%d%s%s%s%s}", esc_file, esc_img, ci, x->line, x->default_of ? ",\"default_of\":" : "", x->default_of ? x->default_of : "", x->build_args ? ",\"build_args\":" : "", x->build_args ? x->build_args : ""); cbm_gbuf_insert_edge(ctx->gbuf, img, mod, "DEPENDS_ON", props); + cbm_free(CBM_MEM_CLASS_OTHER, props); return SKIP_ONE; } diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c index 443bc4961..74ab37a2c 100644 --- a/src/pipeline/pass_terraform.c +++ b/src/pipeline/pass_terraform.c @@ -31,6 +31,7 @@ #include "helpers.h" #include "lang_specs.h" +#include #include #include #include @@ -330,9 +331,96 @@ static char *tf_heredoc(const tf_t *t, TSNode n) { return out; } +/* HCL number literal s (digits, a fraction, an exponent) as Terraform turns a + * number into a string: exact decimal, no exponent, no trailing zeros (cty's + * big.Float Text('f', -1); 1.10 -> "1.1", 2e3 -> "2000"). False when it is + * no such literal or does not fit out. */ +static bool tf_number_text(const char *s, size_t n, char *out, size_t sz) { + enum { TF_MAX_EXP = 4096 }; + char dig[CBM_SZ_256]; + size_t nd = 0; + long exp = 0; + bool frac = false; + size_t i = 0; + for (; i < n && (isdigit((unsigned char)s[i]) || s[i] == '.'); i++) { + if (s[i] == '.') { + if (frac) { + return false; + } + frac = true; + continue; + } + if (nd + SKIP_ONE >= sizeof(dig)) { + return false; + } + dig[nd++] = s[i]; + exp -= frac ? SKIP_ONE : 0; + } + if (i < n) { + if (s[i] != 'e' && s[i] != 'E') { + return false; + } + i++; + bool neg = i < n && s[i] == '-'; + i += i < n && (s[i] == '-' || s[i] == '+'); + long e = 0; + if (i == n) { + return false; + } + for (; i < n; i++) { + if (!isdigit((unsigned char)s[i]) || e > TF_MAX_EXP) { + return false; + } + e = e * 10 + (s[i] - '0'); + } + exp += neg ? -e : e; + } + size_t b = 0; + while (b < nd && dig[b] == '0') { + b++; + } + if (b == nd) { + return snprintf(out, sz, "0") == SKIP_ONE; /* every digit 0 (or none) */ + } + size_t e = nd; + while (dig[e - SKIP_ONE] == '0') { /* stops at dig[b], which is no 0 */ + e--; + exp++; + } + size_t m = e - b; /* significant digits: dig[b, e) x 10^exp */ + size_t o = 0; + long point = (long)m + exp; /* digits before the decimal point */ + size_t need = exp >= 0 ? m + (size_t)exp + : point > 0 ? m + SKIP_ONE + : m + PAIR_LEN + (size_t)-point; + if (need + SKIP_ONE > sz) { + return false; + } + if (point <= 0) { + out[o++] = '0'; + out[o++] = '.'; + for (long z = 0; z < -point; z++) { + out[o++] = '0'; + } + } + for (size_t k = 0; k < m; k++) { + if (point > 0 && (long)k == point) { + out[o++] = '.'; + } + out[o++] = dig[b + k]; + } + for (long z = 0; z < exp; z++) { + out[o++] = '0'; + } + out[o] = '\0'; + return true; +} + /* An HCL literal as JSON: a string without interpolation, a number, a bool, - * null, a tuple or object of those; NULL: anything else. */ -static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { + * null, a tuple or object of those; NULL: anything else. `num_text`: a number + * as the string Terraform makes of it (a helm_release set value is a string), + * else a float64. */ +static yyjson_mut_val *tf_json_v(const tf_t *t, yyjson_mut_doc *doc, TSNode n, bool num_text) { n = tf_unwrap(n); if (tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { char buf[CBM_SZ_4K]; @@ -371,7 +459,11 @@ static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { uint32_t b = ts_node_start_byte(n); uint32_t e = ts_node_end_byte(n); if (tf_type(n, "numeric_lit")) { - char buf[CBM_SZ_64]; + char buf[CBM_SZ_256]; + if (num_text) { + return tf_number_text(t->src + b, e - b, buf, sizeof(buf)) ? yyjson_mut_strcpy(doc, buf) + : NULL; + } snprintf(buf, sizeof(buf), "%.*s", (int)(e - b), t->src + b); return yyjson_mut_real(doc, strtod(buf, NULL)); } @@ -388,7 +480,7 @@ static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { if (tf_type(c, "tuple_start") || tf_type(c, "tuple_end")) { continue; } - yyjson_mut_val *v = tf_json(t, doc, c); + yyjson_mut_val *v = tf_json_v(t, doc, c, num_text); if (!v) { return NULL; } @@ -411,9 +503,9 @@ static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { uint32_t kb = ts_node_start_byte(id); key = yyjson_mut_strncpy(doc, t->src + kb, ts_node_end_byte(id) - kb); } else { - key = tf_json(t, doc, kn); + key = tf_json_v(t, doc, kn, false); } - yyjson_mut_val *val = tf_json(t, doc, vn); + yyjson_mut_val *val = tf_json_v(t, doc, vn, num_text); if (!key || !yyjson_mut_is_str(key) || !val) { return NULL; } @@ -424,38 +516,22 @@ static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { return NULL; } -/* A repository file's text (cbm_alloc), NULL when unreadable. */ +static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { + return tf_json_v(t, doc, n, false); +} + +static char *tf_read(const char *path, size_t *len); + +/* A repository file's text (cbm_alloc; capped and padded like every file + * read), NULL when unreadable -- or when it is no file discovery took in: a + * symlink, or a path through one, is never followed out of the repository. */ static char *tf_read_repo(const tf_t *t, const char *path, size_t *len) { - char full[CBM_SZ_4K]; - snprintf(full, sizeof(full), "%s/%s", t->ctx->repo_path, path); - FILE *f = cbm_fopen(full, "rb"); - if (!f) { + if (!cbm_pipeline_file_node(t->ctx->gbuf, t->ctx->project_name, path)) { return NULL; } - char *buf = NULL; - size_t cap = 0; - char chunk[CBM_SZ_4K]; - size_t r; - *len = 0; - while ((r = fread(chunk, SKIP_ONE, sizeof(chunk), f)) > 0) { - if (*len + r + SKIP_ONE > cap) { - cap = (*len + r + SKIP_ONE) * PAIR_LEN; - char *g = cbm_realloc(CBM_MEM_CLASS_OTHER, buf, cap); - if (!g) { - cbm_free(CBM_MEM_CLASS_OTHER, buf); - (void)fclose(f); - return NULL; - } - buf = g; - } - memcpy(buf + *len, chunk, r); - *len += r; - } - (void)fclose(f); - if (buf) { - buf[*len] = '\0'; - } - return buf; + char full[CBM_SZ_4K]; + int n = snprintf(full, sizeof(full), "%s/%s", t->ctx->repo_path, path); + return n > 0 && (size_t)n < sizeof(full) ? tf_read(full, len) : NULL; } /* A `values = [...]` item as values: a YAML string / heredoc, file() @@ -512,8 +588,8 @@ static yyjson_mut_val *tf_values_item(const tf_t *t, yyjson_mut_doc *doc, TSNode : NULL; } -/* A `set` entry (name / value / type) applied to root; false when it is no - * literal. */ +/* A `set` entry (name / value / type; read with numbers as text) applied to + * root; false when it is no literal. */ static bool tf_set_entry(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_val *entry) { yyjson_mut_val *nm = yyjson_mut_obj_get(entry, "name"); yyjson_mut_val *val = yyjson_mut_obj_get(entry, "value"); @@ -521,13 +597,9 @@ static bool tf_set_entry(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_v if (!yyjson_mut_is_str(nm) || !val) { return false; } - char vbuf[CBM_SZ_64]; const char *v = NULL; if (yyjson_mut_is_str(val)) { v = yyjson_mut_get_str(val); - } else if (yyjson_mut_is_real(val)) { - snprintf(vbuf, sizeof(vbuf), "%.17g", yyjson_mut_get_real(val)); - v = vbuf; } else if (yyjson_mut_is_bool(val)) { v = yyjson_mut_get_bool(val) ? "true" : "false"; } @@ -550,7 +622,7 @@ static yyjson_mut_val *tf_set_block(const tf_t *t, yyjson_mut_doc *doc, TSNode b } TSNode key = ts_node_named_child(at, 0); uint32_t b = ts_node_start_byte(key); - yyjson_mut_val *v = tf_json(t, doc, ts_node_named_child(at, SKIP_ONE)); + yyjson_mut_val *v = tf_json_v(t, doc, ts_node_named_child(at, SKIP_ONE), true); if (!v) { return NULL; } @@ -561,9 +633,29 @@ static yyjson_mut_val *tf_set_block(const tf_t *t, yyjson_mut_doc *doc, TSNode b return entry; } +/* Is block `a` a `dynamic "set"` / `dynamic "set_list"` -- entries made by a + * for_each, never literal? */ +static bool tf_dynamic_set(const tf_t *t, TSNode a) { + TSNode kw = ts_node_named_child(a, 0); + TSNode label = ts_node_named_child(a, SKIP_ONE); + if (!tf_type(a, "block") || ts_node_is_null(kw) || ts_node_is_null(label)) { + return false; + } + uint32_t kb = ts_node_start_byte(kw); + if (ts_node_end_byte(kw) - kb != strlen("dynamic") || + strncmp(t->src + kb, "dynamic", strlen("dynamic")) != 0) { + return false; + } + uint32_t lb = ts_node_start_byte(label); + size_t ll = ts_node_end_byte(label) - lb; + return (ll == strlen("\"set\"") && strncmp(t->src + lb, "\"set\"", ll) == 0) || + (ll == strlen("\"set_list\"") && strncmp(t->src + lb, "\"set_list\"", ll) == 0); +} + /* The values a helm_release body gives its chart -- `values` items * coalesced in order, then each `set` -- as JSON (cbm_alloc); NULL when a - * part is no literal. set_sensitive is never read. */ + * part is no literal, or is made by a `dynamic` block. set_sensitive is + * never read. */ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles) { yyjson_mut_doc *doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; @@ -576,8 +668,10 @@ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, size_t kl = ts_node_is_null(key) ? 0 : ts_node_end_byte(key) - kb; bool is_set = kl == strlen("set") && strncmp(t->src + kb, "set", kl) == 0; bool is_values = kl == strlen("values") && strncmp(t->src + kb, "values", kl) == 0; - /* set_list values are not read: the release's values are partial */ - if (kl == strlen("set_list") && strncmp(t->src + kb, "set_list", kl) == 0) { + /* set_list and dynamic set values are not read: the release's + * values would be partial */ + if ((kl == strlen("set_list") && strncmp(t->src + kb, "set_list", kl) == 0) || + tf_dynamic_set(t, a)) { ok = false; break; } @@ -598,7 +692,7 @@ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, ok = entry && tf_set_entry(doc, root, entry); } else if (pass == SKIP_ONE && is_set && tf_type(a, "attribute")) { /* provider v3: set = [{ name, value, type }, ...] */ - yyjson_mut_val *list = tf_json(t, doc, ts_node_named_child(a, SKIP_ONE)); + yyjson_mut_val *list = tf_json_v(t, doc, ts_node_named_child(a, SKIP_ONE), true); ok = yyjson_mut_is_arr(list); size_t idx, max; yyjson_mut_val *entry; diff --git a/tests/fixtures/complexity/Kubernetes/chart/Chart.yaml b/tests/fixtures/complexity/Kubernetes/chart/Chart.yaml new file mode 100644 index 000000000..7da879473 --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: cx +version: 0.1.0 +appVersion: "1.0" diff --git a/tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl b/tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl new file mode 100644 index 000000000..f4aff46fe --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/templates/_helpers.tpl @@ -0,0 +1,4 @@ +{{- define "cx.image" -}} +{{- $reg := .registry -}} +{{- printf "%s/%s:%s" $reg .repository .tag -}} +{{- end -}} diff --git a/tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml b/tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml new file mode 100644 index 000000000..52c14349c --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/templates/deploy.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cx +spec: + template: + spec: + containers: + - name: main + image: "{{ include "cx.image" .Values.image }}" + {{- range .Values.sidecars }} + - name: {{ .name }} + image: "{{ .image.repository }}:{{ .image.tag }}" + {{- end }} diff --git a/tests/fixtures/complexity/Kubernetes/chart/values.yaml b/tests/fixtures/complexity/Kubernetes/chart/values.yaml new file mode 100644 index 000000000..c279c38a7 --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/chart/values.yaml @@ -0,0 +1,12 @@ +common: &common + pullPolicy: IfNotPresent +image: + <<: *common + registry: ghcr.io + repository: acme/cx + tag: "1.0" +sidecars: + - name: a + image: {repository: ghcr.io/acme/cx-side, tag: "2.0"} + - name: b + image: {repository: ghcr.io/acme/cx-side, tag: "2.1"} diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index f7405e959..2227ac308 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -4766,6 +4766,351 @@ TEST(probe_helm_release_images) { PASS(); } +/* The chart every release test below deploys: an image from two values. */ +#define GPF_APP_CHART \ + {"charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, \ + {"charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, \ + { \ + "charts/app/templates/deploy.yaml", \ + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" \ + " spec:\n containers:\n - name: app\n" \ + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n" \ + } + +/* The edges of the chart's defaults (every test below has them). */ +#define GPF_APP_DEFAULTS \ + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " \ + "default_of=[\"image.repository\",\"image.tag\"]\n" \ + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " \ + "ref=image default_of=[\"image.repository\",\"image.tag\"]" + +/* A Terraform number set on a helm_release reaches the chart as the string + * Terraform makes of it -- its exact decimal (1.1, 1.10 -> "1.1", 2e3 -> + * "2000"), never a float's %.17g ("1.1000000000000001"). */ +TEST(probe_helm_release_terraform_number_set) { + static const GpfFile files[] = { + GPF_APP_CHART, + {"tf/main.tf", "resource \"helm_release\" \"app\" {\n name = \"app\"\n" + " chart = \"../charts/app\"\n set {\n name = \"image.tag\"\n" + " value = 1.10\n }\n}\n"}}; + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.repository\",\"image.tag\"]\n" + "Class:resource.helm_release.app -> Image:ghcr.io/acme/app DEPLOYS " + "image=ghcr.io/acme/app:1.1 via=terraform ref=image override_of=[\"image.tag\"]\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " + "ref=image default_of=[\"image.repository\",\"image.tag\"]"; + if (!gpf_expect_image_edges(files, 4, want)) { + FAIL("a Terraform number set is its exact decimal"); + } + PASS(); +} + +/* A Flux HelmRelease whose values also come from a source not read -- + * valuesFrom, the chart's valuesFiles, a postRenderer that may patch the + * images -- renders no image of its own: its values are not known whole. */ +TEST(probe_helm_release_flux_unread_sources) { + static const GpfFile files[] = { + GPF_APP_CHART, + {"flux/app.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: files\n" + "spec:\n chart:\n spec:\n chart: ./charts/app\n valuesFiles:\n" + " - charts/app/values-prod.yaml\n sourceRef:\n kind: GitRepository\n" + " name: repo\n values:\n image:\n tag: \"9.9\"\n---\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: from\n" + "spec:\n chart:\n spec:\n chart: ./charts/app\n sourceRef:\n" + " kind: GitRepository\n name: repo\n valuesFrom:\n" + " - kind: ConfigMap\n name: app-values\n values:\n image:\n" + " tag: \"8.8\"\n---\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: post\n" + "spec:\n chart:\n spec:\n chart: ./charts/app\n sourceRef:\n" + " kind: GitRepository\n name: repo\n postRenderers:\n" + " - kustomize:\n images:\n - name: ghcr.io/acme/app\n" + " newName: quay.io/acme/app\n values:\n image:\n tag: \"7.7\"\n"}}; + if (!gpf_expect_image_edges(files, 4, GPF_APP_DEFAULTS)) { + FAIL("a HelmRelease with a source not read has no image of its own"); + } + PASS(); +} + +/* A kustomize helmCharts entry with additionalValuesFiles, or a valuesMerge + * other than the default override, renders no image of its own. */ +TEST(probe_helm_release_kustomize_unread_sources) { + static const GpfFile files[] = { + {"kz/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nhelmCharts:\n" + " - name: app\n releaseName: more\n additionalValuesFiles:\n" + " - extra.yaml\n valuesInline:\n image:\n tag: \"5.5\"\n" + " - name: app\n releaseName: merged\n valuesMerge: merge\n" + " valuesInline:\n image:\n tag: \"6.6\"\n"}, + {"kz/charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, + {"kz/charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"kz/charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"}}; + if (!gpf_expect_image_edges(files, 4, GPF_APP_DEFAULTS)) { + FAIL("a helmCharts entry with a part not read has no image of its own"); + } + PASS(); +} + +/* A Terraform helm_release with a `dynamic "set"` block (entries made by a + * for_each) renders no image of its own, whatever its literal sets say. */ +TEST(probe_helm_release_terraform_dynamic_set) { + static const GpfFile files[] = { + GPF_APP_CHART, + {"tf/main.tf", "resource \"helm_release\" \"app\" {\n name = \"app\"\n" + " chart = \"../charts/app\"\n dynamic \"set\" {\n" + " for_each = var.sets\n content {\n name = set.key\n" + " value = set.value\n }\n }\n set {\n name = \"image.tag\"\n" + " value = \"3.3\"\n }\n}\n"}}; + if (!gpf_expect_image_edges(files, 4, GPF_APP_DEFAULTS)) { + FAIL("a helm_release with a dynamic set has no image of its own"); + } + PASS(); +} + +/* A subchart a dependency aliases renders with .Chart.Name its alias (Helm's + * getAliasDependency), so an image naming the chart names the alias. */ +TEST(probe_helm_aliased_subchart_chart_name) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", + "apiVersion: v2\nname: app\nversion: 1.0.0\ndependencies:\n - name: worker\n" + " version: 0.1.0\n alias: worker-high\n"}, + {"charts/app/values.yaml", "worker-high: {}\n"}, + {"charts/app/charts/worker/Chart.yaml", + "apiVersion: v2\nname: worker\nversion: 0.1.0\nappVersion: \"1.0\"\n"}, + {"charts/app/charts/worker/values.yaml", "{}\n"}, + {"charts/app/charts/worker/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: worker\nspec:\n template:\n" + " spec:\n containers:\n - name: worker\n" + " image: \"ghcr.io/acme/{{ .Chart.Name }}:{{ .Chart.AppVersion }}\"\n"}}; + static const char want[] = + "Chart:worker -> Image:ghcr.io/acme/worker-high DEPLOYS image=- via=chart ref=image\n" + "Resource:Deployment/worker -> Image:ghcr.io/acme/worker-high USAGE " + "image=ghcr.io/acme/worker-high:1.0 ref=image"; + if (!gpf_expect_image_edges(files, 5, want)) { + FAIL("an aliased subchart's .Chart.Name is the alias"); + } + PASS(); +} + +/* An image read from a values list keeps its mark on the chart's edge: + * default_of names the list paths (images[0].repository) whole. */ +TEST(probe_helm_list_path_mark_on_chart_edge) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, + {"charts/app/values.yaml", + "images:\n - repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ (index .Values.images 0).repository }}:" + "{{ (index .Values.images 0).tag }}\"\n"}}; + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " + "default_of=[\"images\",\"images[0].repository\",\"images[0].tag\"]\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " + "ref=image default_of=[\"images\",\"images[0].repository\",\"images[0].tag\"]"; + if (!gpf_expect_image_edges(files, 3, want)) { + FAIL("a list path's mark is kept whole on the chart edge"); + } + PASS(); +} + +/* gpf_image_edges with symlinks laid in after the files: links[i] names a + * repository path and the absolute or relative target it points at. */ +static char *gpf_image_edges_linked(const GpfFile *files, int nfiles, const GpfFile *links, + int nlinks) { + GpfProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_gpf_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + return NULL; + } + gpf_to_fwd_slashes(lp.tmpdir); + for (int i = 0; i < nfiles + nlinks; i++) { + const GpfFile *f = i < nfiles ? &files[i] : &links[i - nfiles]; + char path[700]; + snprintf(path, sizeof(path), "%s/%s", lp.tmpdir, f->name); + char *slash = strrchr(path, '/'); + if (slash && slash > path + strlen(lp.tmpdir)) { + *slash = '\0'; + cbm_mkdir_p(path, 0755); + *slash = '/'; + } + if (i >= nfiles) { + if (symlink(f->content, path) != 0) { + return NULL; + } + continue; + } + FILE *fp = fopen(path, "wb"); + if (!fp) { + return NULL; + } + fputs(f->content, fp); + fclose(fp); + } + cbm_store_t *store = gpf_open_indexed(&lp); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT s.label || ':' || s.name " + "|| ' -> Image:' || t.name || ' ' || e.type || ' image=' || " + "coalesce(json_extract(e.properties,'$.image'),'-') || " + "coalesce(' via=' || json_extract(e.properties,'$.via'), '') AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND t.label='Image' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A values file a chart, a kustomization or a Terraform file() names by path + * is read only when it is a file of the repository: a symlink out of it -- + * to a file holding an image -- is never followed. The chart whose values + * file is such a link renders nothing (its values are unknown, not empty); + * the releases reading one have no image of their own. */ +TEST(probe_helm_values_symlink_not_followed) { + char outside[256]; + snprintf(outside, sizeof(outside), "/tmp/cbm_gpf_out_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(outside)); + char leak[300]; + snprintf(leak, sizeof(leak), "%s/values.yaml", outside); + FILE *lf = fopen(leak, "wb"); + ASSERT_NOT_NULL(lf); + fputs("image:\n repository: ghcr.io/evil/leak\n tag: \"6.6\"\n", lf); + fclose(lf); + const GpfFile files[] = { + GPF_APP_CHART, + {"linked/Chart.yaml", "apiVersion: v2\nname: linked\nversion: 1.0.0\n"}, + {"linked/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: linked\nspec:\n template:\n" + " spec:\n containers:\n - name: linked\n" + " image: \"ghcr.io/acme/linked:{{ .Values.tag | default \"latest\" }}\"\n"}, + {"kz/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nhelmCharts:\n" + " - name: app\n releaseName: kapp\n valuesFile: prod.yaml\n"}, + {"kz/charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, + {"kz/charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"kz/charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: kapp\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"}}; + const GpfFile links[] = {{"linked/values.yaml", leak}, {"kz/prod.yaml", leak}}; + char *got = gpf_image_edges_linked(files, 9, links, 2); + th_rmtree(outside); + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart\n" + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0\n" + "Resource:Deployment/kapp -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0"; + bool ok = got && strcmp(got, want) == 0; + if (!ok) { + fprintf(stderr, "image edges got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + } + free(got); + if (!ok) { + FAIL("no values read through a symlink out of the repository"); + } + PASS(); +} + +/* A Terraform helm_release reading its values with file() of a symlink out + * of the repository has no image of its own: the link is never followed. */ +TEST(probe_helm_terraform_values_symlink_not_followed) { + char outside[256]; + snprintf(outside, sizeof(outside), "/tmp/cbm_gpf_out_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(outside)); + char leak[300]; + snprintf(leak, sizeof(leak), "%s/vals.yaml", outside); + FILE *lf = fopen(leak, "wb"); + ASSERT_NOT_NULL(lf); + fputs("image:\n repository: ghcr.io/evil/leak\n tag: \"6.6\"\n", lf); + fclose(lf); + const GpfFile files[] = { + GPF_APP_CHART, + {"tf/main.tf", "resource \"helm_release\" \"app\" {\n name = \"app\"\n" + " chart = \"../charts/app\"\n values = [file(\"vals.yaml\")]\n}\n"}}; + const GpfFile links[] = {{"tf/vals.yaml", leak}}; + char *got = gpf_image_edges_linked(files, 4, links, 1); + th_rmtree(outside); + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0"; + bool ok = got && strcmp(got, want) == 0; + if (!ok) { + fprintf(stderr, "image edges got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + } + free(got); + if (!ok) { + FAIL("no Terraform values read through a symlink out of the repository"); + } + PASS(); +} + +/* A build argument holding a secret -- by its name (*_TOKEN) or its value (a + * GitHub token) -- is never kept on a build edge; the others still choose the + * base image they set. */ +TEST(probe_docker_build_args_keep_no_secret) { + static const GpfFile files[] = { + {"web/Dockerfile", "ARG GO=1.22\nFROM golang:${GO}\n"}, + {"api/Dockerfile", "ARG PY=3.12\nFROM python:${PY}-slim\n"}, + {".env", "NPM_TOKEN=npm-secret-value\n"}, + {"compose.yaml", "services:\n web:\n image: ghcr.io/acme/web\n build:\n" + " context: ./web\n args:\n NPM_TOKEN: ${NPM_TOKEN}\n" + " GO: \"1.23\"\n"}, + {"scripts/build.sh", + "#!/bin/sh\ndocker build --build-arg CI_KEY=ghp_abcdefghijklmnopqrstuvwxyz0123456789AB " + "--build-arg PY=3.13 -t ghcr.io/acme/api:1 api\n"}}; + int secrets = gpf_query_int(files, 5, + "SELECT count(*) FROM edges WHERE project=?1 AND " + "(properties LIKE '%npm-secret-value%' OR properties LIKE " + "'%ghp_abcdefghij%' OR properties LIKE '%NPM_TOKEN%')"); + int overrides = gpf_query_int(files, 5, + "SELECT count(*) FROM edges WHERE project=?1 AND " + "json_extract(properties,'$.override_of') IS NOT NULL"); + ASSERT_EQ(secrets, 0); + ASSERT_EQ(overrides, 2); + PASS(); +} + +/* Build arguments of any length are kept whole on the build edge (never cut + * to malformed JSON): the one choosing the base still makes its edge. */ +TEST(probe_docker_long_build_args_kept_whole) { + enum { ARGS = 40, VAL = 150, CAP = ARGS * (VAL + 40) + 512 }; + char *script = malloc(CAP); + ASSERT_NOT_NULL(script); + int o = snprintf(script, CAP, "#!/bin/sh\ndocker build"); + char val[VAL + 1]; + memset(val, 'v', VAL); + val[VAL] = '\0'; + for (int i = 0; i < ARGS; i++) { + o += snprintf(script + o, (size_t)(CAP - o), " --build-arg LABEL%d=%s", i, val); + } + snprintf(script + o, (size_t)(CAP - o), " --build-arg PY=3.13 -t ghcr.io/acme/api:1 api\n"); + const GpfFile files[] = {{"api/Dockerfile", "ARG PY=3.12\nFROM python:${PY}-slim\n"}, + {"scripts/build.sh", script}}; + int overrides = + gpf_query_int(files, 2, + "SELECT count(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.project=?1 AND t.name='python' AND " + "json_extract(e.properties,'$.override_of') IS NOT NULL"); + free(script); + ASSERT_EQ(overrides, 1); + PASS(); +} + /* A template's `include "name"` calls the define of that name in its chart * family -- the chart and its subcharts share one template namespace -- never * a same-named define of an unrelated chart; a define may also be written in @@ -6580,6 +6925,16 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_docker_default_images_marked_and_overrides); RUN_TEST(probe_helm_template_images_render_with_defaults); RUN_TEST(probe_helm_release_images); + RUN_TEST(probe_helm_release_terraform_number_set); + RUN_TEST(probe_helm_release_flux_unread_sources); + RUN_TEST(probe_helm_release_kustomize_unread_sources); + RUN_TEST(probe_helm_release_terraform_dynamic_set); + RUN_TEST(probe_helm_aliased_subchart_chart_name); + RUN_TEST(probe_helm_list_path_mark_on_chart_edge); + RUN_TEST(probe_helm_values_symlink_not_followed); + RUN_TEST(probe_helm_terraform_values_symlink_not_followed); + RUN_TEST(probe_docker_build_args_keep_no_secret); + RUN_TEST(probe_docker_long_build_args_kept_whole); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 079df09da..e86fd2e77 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4335,6 +4335,230 @@ TEST(helm_values_follow_helm_semantics) { PASS(); } +/* Render template tpl with values src (one probe per `image:` line given) + * and return the capture of probe 0 ("" for UNKNOWN, "" when + * nothing was captured) -- a copy, freed by the caller. */ +static char *helm_capture(const char *vals_src, const char *tpl, const cbm_helm_defines_t *defs, + int line) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *v = doc ? cbm_helm_values_parse(doc, vals_src, strlen(vals_src)) : NULL; + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_4K); + cbm_helm_probe_t probe = {.line = line}; + cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, NULL, defs, &probe, 1); + char *out = strdup(probe.n == 0 ? "" : probe.image[0] ? probe.image[0] : ""); + cbm_arena_destroy(&a); + if (doc) { + yyjson_mut_doc_free(doc); + } + return out; +} + +/* A render doing far more work than any chart -- here a range over an int + * of 20,000,000 -- stops at its step budget (helm_template.c): every capture + * of the template is UNKNOWN, also the one made before the loop, never a + * partial or wrong image; a range over a huge int ends the same way. */ +TEST(helm_render_range_budget_ends_unknown) { + static const char vals[] = "tag: \"1.0\"\n"; + static const char ok[] = "image: \"ghcr.io/acme/app:{{ .Values.tag }}\"\n"; + char *got = helm_capture(vals, ok, NULL, 1); + ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + free(got); + static const char rng[] = "image: \"ghcr.io/acme/app:{{ .Values.tag }}\"\n" + "{{- range 20000000 }}{{ end }}\n"; + got = helm_capture(vals, rng, NULL, 1); + ASSERT_STR_EQ(got, ""); + free(got); + static const char huge[] = "image: \"ghcr.io/acme/app:{{ .Values.tag }}\"\n" + "{{- range 2000000000 }}{{ range 2000000000 }}{{ end }}{{ end }}\n"; + got = helm_capture(vals, huge, NULL, 1); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* An indent whose output passes the output budget stops the render: no + * capture of it is known (an indent of 9e18 stops before any work). */ +TEST(helm_render_indent_budget_ends_unknown) { + static const char big[] = "image: \"ghcr.io/acme/app{{ indent 70000000 \"\" | trim }}:1\"\n"; + char *got = helm_capture("{}\n", big, NULL, 1); + ASSERT_STR_EQ(got, ""); + free(got); + static const char huge[] = + "image: \"ghcr.io/acme/app{{ indent 9000000000000000000 \"\" | trim }}:1\"\n"; + got = helm_capture("{}\n", huge, NULL, 1); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* Defines that each include the one below twice multiply the work by two a + * level: past the step budget the render stops, its captures UNKNOWN. */ +TEST(helm_render_define_fanout_ends_unknown) { + enum { LEVELS = 21, FAN_CAP = 8192 }; + char *fan = malloc(FAN_CAP); + ASSERT_NOT_NULL(fan); + int o = snprintf(fan, FAN_CAP, "{{- define \"d0\" -}}x{{- end -}}\n"); + for (int k = 1; k < LEVELS; k++) { + o += snprintf(fan + o, (size_t)(FAN_CAP - o), + "{{- define \"d%d\" -}}{{ include \"d%d\" . }}{{ include \"d%d\" . }}" + "{{- end -}}\n", + k, k - 1, k - 1); + } + cbm_helm_defines_t *d = cbm_helm_defines_new(); + ASSERT_NOT_NULL(d); + cbm_helm_defines_add(d, "templates/_fan.tpl", fan, strlen(fan)); + free(fan); + static const char tpl[] = "image: \"ghcr.io/acme/app:{{ include \"d20\" . | trunc 1 }}\"\n"; + char *got = helm_capture("{}\n", tpl, d, 1); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* go-yaml v2's aliasing limit (Helm reads values with it): a document whose + * decodes come mostly from alias expansion -- a billion laughs -- is no + * values; a few aliases still read. */ +TEST(helm_values_alias_expansion_limit) { + enum { LEVELS = 5, BUF = 4096 }; + char laughs[BUF]; + int o = snprintf(laughs, sizeof(laughs), "a0: &a0 [x, x, x, x, x, x, x, x, x, x]\n"); + for (int k = 1; k < LEVELS; k++) { + o += snprintf(laughs + o, sizeof(laughs) - (size_t)o, "a%d: &a%d [", k, k); + for (int i = 0; i < 10; i++) { + o += snprintf(laughs + o, sizeof(laughs) - (size_t)o, "%s*a%d", i ? ", " : "", k - 1); + } + o += snprintf(laughs + o, sizeof(laughs) - (size_t)o, "]\n"); + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + ASSERT_NOT_NULL(doc); + static const char few[] = "base: &b {tag: \"1\"}\none: *b\ntwo: *b\n"; + yyjson_mut_val *ok = cbm_helm_values_parse(doc, few, strlen(few)); + yyjson_mut_val *bomb = cbm_helm_values_parse(doc, laughs, strlen(laughs)); + yyjson_mut_doc_free(doc); + ASSERT_NOT_NULL(ok); + ASSERT_NULL(bomb); + PASS(); +} + +/* --set as Helm's strvals reads it: a list index past MaxIndex (65536) is an + * error, so the set is no value (a release with it is unknown) -- never a + * list padded to it; a key longer than the parser holds is no value either, + * never split into two keys. */ +TEST(helm_values_set_index_cap_and_long_key) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + ASSERT_NOT_NULL(doc); + yyjson_mut_val *root = yyjson_mut_obj(doc); + ASSERT_TRUE(cbm_helm_values_set(doc, root, "list[3]", "x", false)); + ASSERT_FALSE(cbm_helm_values_set(doc, root, "big[65537]", "x", false)); + ASSERT_FALSE(cbm_helm_values_set(doc, root, "huge[9000000000000]", "x", false)); + char key[700]; + memset(key, 'k', sizeof(key) - 1); + key[sizeof(key) - 1] = '\0'; + ASSERT_FALSE(cbm_helm_values_set(doc, root, key, "x", false)); + yyjson_mut_doc_free(doc); + PASS(); +} + +/* A values key longer than 1 KiB read by an image is in its default_of + * mark whole -- a mark is never cut. */ +TEST(helm_render_long_key_mark_whole) { + enum { KEY = 1500 }; + char *key = malloc(KEY + 1); + ASSERT_NOT_NULL(key); + memset(key, 'k', KEY); + key[KEY] = '\0'; + size_t vn = KEY + CBM_SZ_64; + char *vals = malloc(vn); + char *tpl = malloc(vn); + ASSERT_NOT_NULL(vals); + ASSERT_NOT_NULL(tpl); + snprintf(vals, vn, "%s: \"1.0\"\n", key); + snprintf(tpl, vn, "image: \"ghcr.io/acme/app:{{ index .Values \"%s\" }}\"\n", key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *v = cbm_helm_values_parse(doc, vals, strlen(vals)); + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_4K); + cbm_helm_probe_t probe = {.line = 1}; + cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, NULL, NULL, &probe, 1); + char want[KEY + 8]; + snprintf(want, sizeof(want), "[\"%s\"]", key); + bool whole = probe.n == 1 && probe.keys[0] && strcmp(probe.keys[0], want) == 0; + cbm_arena_destroy(&a); + yyjson_mut_doc_free(doc); + free(key); + free(vals); + free(tpl); + ASSERT_TRUE(whole); + PASS(); +} + +/* A condition that is UNKNOWN runs no branch -- and what a branch would + * assign is UNKNOWN after it: `$tag = .Values.image.tagNew` under a + * .Capabilities test leaves no image naming the old tag. */ +TEST(helm_unknown_branch_spoils_what_it_assigns) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\", tagNew: \"2.0\"}\n"; + static const char tpl[] = + "{{- $tag := .Values.image.tag -}}\n" + "{{- if semverCompare \">=1.29-0\" .Capabilities.KubeVersion.Version }}" + "{{- $tag = .Values.image.tagNew -}}{{- end }}\n" + "image: \"{{ .Values.image.repository }}:{{ $tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* set / unset / merge change a map in place -- not modelled: what it holds + * after is UNKNOWN (an image read before the change keeps its value). */ +TEST(helm_mutator_spoils_the_map_it_changes) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + static const char tpl[] = "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n" + "{{- $_ := set .Values.image \"tag\" \"2.0\" }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *before = helm_capture(vals, tpl, NULL, 1); + char *after = helm_capture(vals, tpl, NULL, 3); + ASSERT_STR_EQ(before, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(after, ""); + free(before); + free(after); + PASS(); +} + +/* A break under an UNKNOWN condition in a range: whether the loop goes on + * is unknown, so its later iterations are not rendered (Helm with a true + * condition renders no image here). */ +TEST(helm_unknown_break_leaves_the_loop) { + static const char tpl[] = "{{- range list \"a\" \"b\" }}\n" + "{{- if .Capabilities.APIVersions }}{{ break }}{{ end }}\n" + "image: \"ghcr.io/acme/{{ . }}:1\"\n" + "{{ end }}\n"; + char *got = helm_capture("{}\n", tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A change made to a new value -- fromYaml / deepCopy results, a part of one, + * an empty dict -- reaches nothing another part holds: the images after it + * stay known (the opentelemetry-ebpf-instrumentation configData pattern). */ +TEST(helm_change_of_a_new_value_spoils_nothing) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\ncfg: {a: 1}\n"; + static const char tpl[] = + "{{- $config := dict -}}\n" + "{{- $user := deepCopy (.Values.cfg | default dict) -}}\n" + "{{- if .Capabilities.APIVersions }}{{- $config = toYaml .Values.cfg | fromYaml -}}" + "{{- $config = mergeOverwrite $config $user -}}{{- end }}\n" + "{{- $config = mergeOverwrite $config $user -}}\n" + "{{- $_ := unset (deepCopy .Values.cfg) \"a\" }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 6); + ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + free(got); + PASS(); +} + /* A compose project's .env feeds its compose file (variables in image, * build): changing it is no closure repair -- the unchanged compose file's * services would keep the old image. It rebuilds; no stale image remains. */ @@ -24063,6 +24287,16 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_werf_values_edit_keeps_chart_edge); RUN_TEST(incremental_release_values_edit_rebuilds); RUN_TEST(helm_values_follow_helm_semantics); + RUN_TEST(helm_render_range_budget_ends_unknown); + RUN_TEST(helm_render_indent_budget_ends_unknown); + RUN_TEST(helm_render_define_fanout_ends_unknown); + RUN_TEST(helm_values_alias_expansion_limit); + RUN_TEST(helm_values_set_index_cap_and_long_key); + RUN_TEST(helm_render_long_key_mark_whole); + RUN_TEST(helm_unknown_branch_spoils_what_it_assigns); + RUN_TEST(helm_mutator_spoils_the_map_it_changes); + RUN_TEST(helm_unknown_break_leaves_the_loop); + RUN_TEST(helm_change_of_a_new_value_spoils_nothing); RUN_TEST(incremental_compose_dotenv_change_rebuilds); RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges); RUN_TEST(incremental_second_build_declarer_keeps_the_edge); From 033be76c703928a9ee2a8e74eec2ee26f2674c1f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 18 Oct 2026 21:21:37 +0200 Subject: [PATCH 09/19] fix(doc-links): a Go method claims no flat doc-link node name #1913 makes Go method nodes receiver-qualified (..). The Go doc-link model still let every method claim the flat node .: - doclink_go.c go_scan_method pushed a claim record; - doc_links_go.c go_add_claim put it in the flat table. So a type, function or value that shares its name with a method in its directory became a graph_gap ("lost its node to the method"), although the graph now gives it that node. Base reports the same graph_gap for this shape, because there the method really did take the node. So this completes #1913's QN scheme for one of its consumers; it is not a regression #1913 causes. - A method record now carries w `-`: it claims no flat name. - The resolver no longer enters methods into the flat table. Methods are found through their receiver (the member table), which was already the first lookup. - The flat fallback in go_method_node is now unreachable. It stays, with a comment saying so. Corpus, before = this branch's tree, par x2, r1 == r2: - terraform: graph_gap rows 9 -> 0, doc-link MENTIONS 528 -> 536 (+8, -0). All 8 new edges were judged correct: e.g. [AbsProviderConfig] now lands on the type, not on the method RootProviderConfig.AbsProviderConfig. - vault, prometheus: unchanged. - prometheus incremental == full for MENTIONS. - The 3 -> 4 index-format bump already forces a full rebuild, so no stored scope blob mixes the two meanings. Signed-off-by: Martin Vogel --- internal/cbm/doclink_go.c | 16 +++++++++------- src/pipeline/doc_links_go.c | 23 +++++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/internal/cbm/doclink_go.c b/internal/cbm/doclink_go.c index 97dab618d..aeb0c5858 100644 --- a/internal/cbm/doclink_go.c +++ b/internal/cbm/doclink_go.c @@ -43,10 +43,10 @@ * drops these lines and so carries no line number * at all. * G path go.mod: the module path - * w says who keeps the node. A Go method node has a FLAT qualified name - * (.), so a function, the methods of every receiver, a type and - * a value of one name all claim the node ., and within a file the - * declaration on the LAST line keeps it (cbm_gbuf_upsert_node). w is 1 for + * w says who keeps the node .. A function, a type and a value of + * one name all claim it, and within a file the declaration on the LAST line + * keeps it (cbm_gbuf_upsert_node). A method claims nothing: its node is + * receiver-qualified (.., #1913), and its w is `-`. w is 1 for * that declaration, 0 for the others of that name, x when two of them start * on that last line, and - for a value the extractor gives no node at all * (it gives one to the first name of a constant spec and of a parenthesis- @@ -1511,9 +1511,11 @@ static void go_scan_method(go_scan_t *s, TSNode m) { recv = "?"; recv_len = SKIP_ONE; } - go_push_claim(s, - cbm_arena_sprintf(s->tmp, "M\t%.*s\t%.*s", (int)recv_len, recv, (int)len, name), - name, len, m); + /* #1913: a Go method node is receiver-qualified (..), + * so a method claims no flat name: w is `-`, and the method is found + * through its receiver (the member table), never through .. */ + (void)go_push_rec( + s, cbm_arena_sprintf(s->tmp, "M\t%.*s\t%.*s\t-", (int)recv_len, recv, (int)len, name)); } /* The implied name of an embedded field: the type's own name, without diff --git a/src/pipeline/doc_links_go.c b/src/pipeline/doc_links_go.c index 047a7654e..b43289c95 100644 --- a/src/pipeline/doc_links_go.c +++ b/src/pipeline/doc_links_go.c @@ -32,10 +32,11 @@ * fields of its own struct declaration and the methods of its own interface * declaration: nothing promoted through embedding. * - * Binding. The graph keeps one node per qualified name, Go method nodes have - * a FLAT name (.), and is the directory, not the package - * clause: functions, methods of every receiver, types and values of one name, - * in every file of the directory, share one node, and the node is ONE of those + * Binding. The graph keeps one node per qualified name, and is the + * directory, not the package clause. Go method nodes are receiver-qualified + * (.., #1913) and are found through their receiver only. + * Functions, types and values of one name, in every file of the directory, + * share one node ., and the node is ONE of those * declarations (cbm_gbuf_upsert_node keeps the smallest file path, then the * last line). So a reference binds only when the node IS the entity it names: * the node's file is a file of the named package, the declaration that keeps @@ -59,7 +60,7 @@ * not), missing (a repository package that does not declare the name; an * import path inside a module without a package there), graph_gap (declared, * but the node is another declaration's or absent: block variables, later - * names of a declaration, embedded fields, methods that lost the flat name; + * names of a declaration, embedded fields; * also an undeclared name in a package with a parse error), test_only_target * (product code naming a declaration that only test files have), ambiguous * (several modules or packages fit; a name declared as two kinds; a name two @@ -1063,7 +1064,7 @@ static void go_record_decl(go_index_t *ix, go_file_t *file, char **f, int nf) { FIELD_FIELDS = 4, IM_FIELDS = 3 }; - enum { F_OWNER = 2, T_ALIAS = 3, T_OWNER = 4, V_OWNER = 2, M_OWNER = 3, FIELD_NODE = 3 }; + enum { F_OWNER = 2, T_ALIAS = 3, T_OWNER = 4, V_OWNER = 2, FIELD_NODE = 3 }; char kind = f[0][0]; if (nf < IM_FIELDS || !go_name_ok(f[SKIP_ONE])) { return; @@ -1078,10 +1079,12 @@ static void go_record_decl(go_index_t *ix, go_file_t *file, char **f, int nf) { go_add_sym(ix, file, GO_KIND_VALUE, f[SKIP_ONE], NULL); go_add_claim(ix, file, GO_KIND_VALUE, f[SKIP_ONE], NULL, f[V_OWNER]); } else if (kind == 'M' && nf == M_FIELDS && go_name_ok(f[PAIR_LEN])) { + /* #1913: a method is found through its receiver only; it claims no + * flat . (its record says w `-`), so a type, function or + * value of the same name keeps that node. */ if (strcmp(f[SKIP_ONE], "?") != 0) { go_add_member(ix, file, GO_MEM_METHOD, f[SKIP_ONE], f[PAIR_LEN], true); } - go_add_claim(ix, file, GO_KIND_METHOD, f[PAIR_LEN], f[SKIP_ONE], f[M_OWNER]); } else if (kind == 'f' && nf == FIELD_FIELDS && go_name_ok(f[PAIR_LEN])) { go_add_member(ix, file, GO_MEM_FIELD, f[SKIP_ONE], f[PAIR_LEN], f[FIELD_NODE][0] == '1'); } else if (kind == 'm' && nf == IM_FIELDS && go_name_ok(f[PAIR_LEN])) { @@ -1342,9 +1345,9 @@ static const cbm_gbuf_node_t *go_member_node(const go_ctx_t *c, const go_pkg_t * return (fm && (fm->node_kinds & kind)) ? n : NULL; } -/* The node of the method `recv.name`: its receiver-qualified node, else the - * flat node . when the method is the one that kept it (the last - * declaration of that name in the node's file). */ +/* The node of the method `recv.name`: its receiver-qualified node. The flat + * fallback below (the method that kept .) is unreachable since + * #1913: no method record claims a flat name any more. */ static const cbm_gbuf_node_t *go_method_node(const go_ctx_t *c, const go_pkg_t *pkg, go_str_t recv, go_str_t name) { const cbm_gbuf_node_t *qualified = go_member_node(c, pkg, GO_MEM_METHOD, recv, name); From 4c62b61845d4e9e402c4bf397311bf7b23ccb7b7 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 18 Oct 2026 21:21:37 +0200 Subject: [PATCH 10/19] test(doc_mentions_go): receiver-qualified Go method nodes (#1913) main's Go doc-link tests were written against flat Go method QNs, where `(A) Close` and `(B) Close` shared one node `alpha.Close`. That is the fused-node defect #1913 removes. The expectations move to the new scheme, and each changed assertion pins an edge onto, or a count of, the receiver-qualified node (production: the previous commit and #1913): - sources: RefM's source is p.pkg.S.M, not p.pkg.M. - forms: [Widget.Run] lands on alpha.Widget.Run, [beta.Thing.Do] on beta.Thing.Do. - plain_text, struct_fields: Run's doc is the source alpha.Widget.Run. - lookup_order: two alpha.Run sources and beta.Promoted named no node any more, so they held vacuously. They are now alpha.Widget.Run and beta.Embedded.Promoted. The alias links land on beta.Thing.Do and alpha.Deep10.End. - node_owner, the positive contract that replaces the fused case: - two Method nodes, alpha.A.Close and alpha.B.Close, and none named alpha.Close; - [A.Close] and [B.Close] are edges, not graph_gap rows; - each method's own [Target] is an edge from its own doc line (9 and 12), with no unresolved row; - the type Open keeps alpha.Open: [Open] is the type (no graph_gap), [B.Open] is the method, and UsesOpen has 2 targets. - scope_blob: - method records carry w `-`; - swapping two methods of one name no longer changes the blob; - a func/type pair swap still does (new case); - the k1-k3 records follow (a method neither keeps nor contests a name); - a new k4 keeps the "two on one line contest it" rule covered with a var and a const. - incremental_equals_full: - [b.T.M], [b.W.Name] and [b.V.Name] all resolve, and U.M's [Target] is an edge; - removing T.M breaks only [b.T.M]; - swapping V.Name and W.Name moves nothing and is now an ordinary closure repair instead of a forced full rebuild. - scope_delta_rules: - a method/function swap is LOCAL; - removing the method of such a pair is LOCAL with the name "Open" (a member left); - new func/type cases keep the GLOBAL keeper-change rules covered for swap and removal. - workers_agree: [p01.T3.Close] now resolves (exact edge, no row). The graph_gap reason stays covered by the named Shared assertion: one function declared by every file of a package, which is still a gap. Signed-off-by: Martin Vogel --- tests/test_doc_mentions_go.c | 190 ++++++++++++++++++++++------------- 1 file changed, 122 insertions(+), 68 deletions(-) diff --git a/tests/test_doc_mentions_go.c b/tests/test_doc_mentions_go.c index 78f1934e0..0d4462f76 100644 --- a/tests/test_doc_mentions_go.c +++ b/tests/test_doc_mentions_go.c @@ -228,10 +228,10 @@ TEST(doc_mentions_go_sources) { const char *raw; const char *source; } want[] = { - {"RefF", "p.pkg.F"}, {"RefS", "p.pkg.S"}, {"RefField", "p.pkg.S.Field"}, - {"RefM", "p.pkg.M"}, /* Go method nodes have a flat name */ - {"RefI", "p.pkg.I"}, {"RefDo", "p.pkg.I.Do"}, {"RefC", "p.pkg.C"}, - {"RefK", "p.pkg.K"}, {"RefV", "p.pkg.V"}, {"RefA", "p.pkg.A"}, + {"RefF", "p.pkg.F"}, {"RefS", "p.pkg.S"}, {"RefField", "p.pkg.S.Field"}, + {"RefM", "p.pkg.S.M"}, /* #1913: Go method nodes are receiver-qualified */ + {"RefI", "p.pkg.I"}, {"RefDo", "p.pkg.I.Do"}, {"RefC", "p.pkg.C"}, + {"RefK", "p.pkg.K"}, {"RefV", "p.pkg.V"}, {"RefA", "p.pkg.A"}, }; for (size_t i = 0; i < GO_COUNT(want); i++) { const CBMDocLink *t = dm_find_token(r, want[i].raw); @@ -772,14 +772,15 @@ TEST(doc_mentions_go_scope_blob) { "T\tAlias\ta\tal.Widget\t1", "T\tSame\ta\tKind\t1", "T\tList\ts\t-\t1", - /* functions and methods; two methods named Run compete for one node - * and the later one keeps it */ + /* functions and methods; a method node is receiver-qualified + * (#1913), so a method claims no flat name and its w is `-`: the two + * methods named Run no longer compete */ "F\tNew\t1", "F\tMap\t1", - "M\tWidget\tRun\t0", - "M\tKind\tRun\t1", - "M\tList\tPush\t1", - "M\tlower\tExported\t1", + "M\tWidget\tRun\t-", + "M\tKind\tRun\t-", + "M\tList\tPush\t-", + "M\tlower\tExported\t-", /* own type parameters, by the declaration's first and last line */ "g\t36\t36\tT,U", "g\t42\t42\tA,B", @@ -814,15 +815,20 @@ TEST(doc_mentions_go_scope_blob) { char *portable = cbm_doclink_portable_scope(s); ASSERT_NOT_NULL(portable); ASSERT_NULL(strstr(portable, "\ng\t")); - ASSERT_TRUE(go_has_rec(portable, "M\tKind\tRun\t1")); + ASSERT_TRUE(go_has_rec(portable, "M\tKind\tRun\t-")); ASSERT(strncmp(portable, "go1\n", 4) == 0); cbm_free(CBM_MEM_CLASS_OTHER, portable); cbm_free_result(r); - /* moving declarations changes nothing unless a node changes its owner */ + /* moving declarations changes nothing unless a node changes its owner. + * Methods own no flat name (#1913), so swapping two methods of one name + * changes nothing either; two package-level declarations of one name + * still hand . over when swapped */ const char *a = "package p\n\nfunc (a A) Run() {}\nfunc (b B) Run() {}\nfunc F() {}\n"; const char *moved = "package p\n\nfunc F() {}\n\n\nfunc (a A) Run() {}\nfunc (b B) Run() {}\n"; const char *swapped = "package p\n\nfunc (b B) Run() {}\nfunc (a A) Run() {}\nfunc F() {}\n"; + const char *pair = "package p\n\nfunc Open() {}\ntype Open struct{}\n"; + const char *pair_swapped = "package p\n\ntype Open struct{}\nfunc Open() {}\n"; CBMFileResult *ra = go_extract(a, "p/x.go"); CBMFileResult *rm = go_extract(moved, "p/x.go"); CBMFileResult *rs = go_extract(swapped, "p/x.go"); @@ -830,16 +836,25 @@ TEST(doc_mentions_go_scope_blob) { ASSERT_NOT_NULL(rm); ASSERT_NOT_NULL(rs); ASSERT_STR_EQ(ra->doc_scope, rm->doc_scope); - ASSERT(strcmp(ra->doc_scope, rs->doc_scope) != 0); - ASSERT_TRUE(go_has_rec(rs->doc_scope, "M\tA\tRun\t1")); - ASSERT_TRUE(go_has_rec(rs->doc_scope, "M\tB\tRun\t0")); + ASSERT_STR_EQ(ra->doc_scope, rs->doc_scope); + ASSERT_TRUE(go_has_rec(rs->doc_scope, "M\tA\tRun\t-")); + ASSERT_TRUE(go_has_rec(rs->doc_scope, "M\tB\tRun\t-")); cbm_free_result(ra); cbm_free_result(rm); cbm_free_result(rs); + CBMFileResult *rp = go_extract(pair, "p/x.go"); + CBMFileResult *rq = go_extract(pair_swapped, "p/x.go"); + ASSERT_NOT_NULL(rp); + ASSERT_NOT_NULL(rq); + ASSERT(strcmp(rp->doc_scope, rq->doc_scope) != 0); + ASSERT_TRUE(go_has_rec(rp->doc_scope, "T\tOpen\ts\t-\t1")); + ASSERT_TRUE(go_has_rec(rq->doc_scope, "F\tOpen\t1")); + cbm_free_result(rp); + cbm_free_result(rq); - /* the node . is one for a function, a type, a value and the - * methods of that name: the last declaration keeps it, whatever its kind, - * and two on one line contest it */ + /* the node . is one for a function, a type and a value of + * that name: the last declaration keeps it, whatever its kind, and two on + * one line contest it. A method of that name claims nothing (#1913) */ CBMFileResult *k1 = go_extract("package p\n\nfunc Open() {}\nfunc (t T) Open() {}\n", "p/k.go"); CBMFileResult *k2 = go_extract("package p\n\nfunc (t T) Open() {}\nfunc Open() {}\n", "p/k.go"); CBMFileResult *k3 = go_extract("package p\n\ntype Open int\nvar Open = 1\n" @@ -848,14 +863,21 @@ TEST(doc_mentions_go_scope_blob) { ASSERT_NOT_NULL(k1); ASSERT_NOT_NULL(k2); ASSERT_NOT_NULL(k3); - ASSERT_TRUE(go_has_rec(k1->doc_scope, "F\tOpen\t0")); - ASSERT_TRUE(go_has_rec(k1->doc_scope, "M\tT\tOpen\t1")); + ASSERT_TRUE(go_has_rec(k1->doc_scope, "F\tOpen\t1")); + ASSERT_TRUE(go_has_rec(k1->doc_scope, "M\tT\tOpen\t-")); ASSERT_TRUE(go_has_rec(k2->doc_scope, "F\tOpen\t1")); - ASSERT_TRUE(go_has_rec(k2->doc_scope, "M\tT\tOpen\t0")); + ASSERT_TRUE(go_has_rec(k2->doc_scope, "M\tT\tOpen\t-")); ASSERT_TRUE(go_has_rec(k3->doc_scope, "T\tOpen\to\t-\t0")); ASSERT_TRUE(go_has_rec(k3->doc_scope, "V\tOpen\t0")); - ASSERT_TRUE(go_has_rec(k3->doc_scope, "V\tOpen\tx")); - ASSERT_TRUE(go_has_rec(k3->doc_scope, "M\tT\tOpen\tx")); + ASSERT_TRUE(go_has_rec(k3->doc_scope, "V\tOpen\t1")); + ASSERT_TRUE(go_has_rec(k3->doc_scope, "M\tT\tOpen\t-")); + /* two package-level declarations on one line contest the name */ + CBMFileResult *k4 = + go_extract("package p\n\ntype Open int\nvar Open = 1; const Open = 2\n", "p/k.go"); + ASSERT_NOT_NULL(k4); + ASSERT_TRUE(go_has_rec(k4->doc_scope, "T\tOpen\to\t-\t0")); + ASSERT_TRUE(go_has_rec(k4->doc_scope, "V\tOpen\tx")); + cbm_free_result(k4); cbm_free_result(k1); cbm_free_result(k2); cbm_free_result(k3); @@ -1030,7 +1052,7 @@ static const char GO_ALPHA_C[] = "package alpha\n" "// Target is a target; see [A.Close] and [B.Close].\n" "func Target() {}\n" "\n" - "// Open is a type that loses its node name to the method below.\n" + "// Open is a type; a method of B below has the same name.\n" "type Open struct{}\n" "\n" "// Open opens: a method named like the type.\n" @@ -1295,7 +1317,7 @@ TEST(doc_mentions_go_forms) { static const go_want_edge_t edges[] = { /* [Name], [Recv.Method], an interface's method, a constant, a variable */ {"alpha.Widget", "alpha.New", "unique"}, - {"alpha.Widget", "alpha.Run", "unique"}, + {"alpha.Widget", "alpha.Widget.Run", "unique"}, {"alpha.Widget", "alpha.Runner.Run", "unique"}, {"alpha.Widget", "alpha.Max", "unique"}, {"alpha.Widget", "alpha.Default", "unique"}, @@ -1311,7 +1333,7 @@ TEST(doc_mentions_go_forms) { {"alpha.New", "sub.deep.Deep", "exact"}, {"alpha.New", "nested.pkg.Far", "exact"}, /* [pkg.Recv.Name]: a method, a field, an interface method */ - {"alpha.New", "beta.Do", "exact"}, + {"alpha.New", "beta.Thing.Do", "exact"}, {"alpha.New", "beta.Thing.Field", "exact"}, {"alpha.New", "beta.Doer.Act", "exact"}, /* an identifier with letters beyond ASCII */ @@ -1395,7 +1417,7 @@ TEST(doc_mentions_go_plain_text) { ASSERT_EQ(go_bad_rows(db, rows, GO_COUNT(rows)), 0); /* and none of them is an edge: Run's doc holds rows, plain text and one * link, to the field Widget.Size */ - ASSERT_EQ(dm_mentions_from(db, "alpha.Run"), 1); + ASSERT_EQ(dm_mentions_from(db, "alpha.Widget.Run"), 1); ASSERT_EQ(dm_mentions_from(db, "alpha.Bare"), 0); static const go_want_edge_t edges[] = { {"alpha.Sibling", "alpha.Widget", "unique"}, @@ -1424,14 +1446,14 @@ TEST(doc_mentions_go_struct_fields) { static const go_want_edge_t edges[] = { /* the same package: a field named from a method's doc and from a * sibling field's doc */ - {"alpha.Run", "alpha.Widget.Size", "exact"}, + {"alpha.Widget.Run", "alpha.Widget.Size", "exact"}, {"alpha.Widget.Name", "alpha.Widget.Size", "exact"}, /* another package, by its import */ {"alpha.New", "beta.Thing.Field", "exact"}, {"gamma.ViaAlias", "alpha.Widget.Name", "exact"}, /* Depth is a field of beta.Embedded, which Widget embeds: it is no * field of Widget */ - {"alpha.Run", "beta.Embedded.Depth", ""}, + {"alpha.Widget.Run", "beta.Embedded.Depth", ""}, }; ASSERT_EQ(go_bad_edges(db, edges, GO_COUNT(edges)), 0); /* the promoted field exists as a node: the test above is not vacuous */ @@ -1545,7 +1567,7 @@ TEST(doc_mentions_go_lookup_order) { * file that imports "fmt", the repository's own in a file that imports * example.com/demo/fmt */ ASSERT_STR_EQ(go_reason(db, "alpha/a.go", "fmt.Println"), "external"); - ASSERT_STR_EQ(go_tier(db, "alpha.Run", "fmt.Println"), ""); + ASSERT_STR_EQ(go_tier(db, "alpha.Widget.Run", "fmt.Println"), ""); ASSERT_STR_EQ(go_tier(db, "gamma.Local", "fmt.Println"), "exact"); /* 5. the longest module path decides: a directory inside the root module * that belongs to a nested module of another path is not reachable through @@ -1555,17 +1577,17 @@ TEST(doc_mentions_go_lookup_order) { ASSERT_STR_EQ(go_tier(db, "alpha.New", "sub.deep.Deep"), "exact"); /* 6. the bare name stops at the package: beta.Make is imported and still * no [Make] */ - ASSERT_STR_EQ(go_tier(db, "alpha.Run", "beta.Make"), ""); + ASSERT_STR_EQ(go_tier(db, "alpha.Widget.Run", "beta.Make"), ""); /* 7. members stop at the type: nothing promoted through embedding */ ASSERT_STR_EQ(go_reason(db, "gamma/k.go", "alpha.Widget.Promoted"), "missing"); - ASSERT_STR_EQ(go_tier(db, "gamma.ViaAlias", "beta.Promoted"), ""); + ASSERT_STR_EQ(go_tier(db, "gamma.ViaAlias", "beta.Embedded.Promoted"), ""); /* 8. an alias is followed for a member written with its package, to the * type that declares the member */ static const go_want_edge_t alias[] = { - {"gamma.ViaAlias", "beta.Do", "exact"}, + {"gamma.ViaAlias", "beta.Thing.Do", "exact"}, {"gamma.ViaAlias", "beta.Thing.Field", "exact"}, {"gamma.ViaAlias", "alpha.Alias", "exact"}, - {"gamma.ViaAlias", "alpha.End", "exact"}, /* seven aliases away */ + {"gamma.ViaAlias", "alpha.Deep10.End", "exact"}, /* seven aliases away */ {"gamma.ViaAlias", "alpha.Widget.Name", "exact"}, }; ASSERT_EQ(go_bad_edges(db, alias, GO_COUNT(alias)), 0); @@ -1638,28 +1660,42 @@ TEST(doc_mentions_go_node_owner) { char tmp[256]; char db[512]; ASSERT_EQ(go_index_lookup(tmp, sizeof(tmp), db, sizeof(db)), 0); - /* Go method nodes have a flat name: A.Close and B.Close share - * alpha.Close, and the later declaration keeps it */ - ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE qualified_name LIKE '%.alpha.Close'"), + /* #1913: Go method nodes are receiver-qualified. A.Close and B.Close are + * two nodes (no shared alpha.Close), each a target of its own */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'Method' AND " + "qualified_name LIKE '%.alpha.A.Close'"), 1); - ASSERT_STR_EQ(go_tier(db, "alpha.Target", "alpha.Close"), "unique"); - ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "A.Close"), "graph_gap"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'Method' AND " + "qualified_name LIKE '%.alpha.B.Close'"), + 1); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE qualified_name LIKE '%.alpha.Close'"), + 0); + ASSERT_STR_EQ(go_tier(db, "alpha.Target", "alpha.A.Close"), "unique"); + ASSERT_STR_EQ(go_tier(db, "alpha.Target", "alpha.B.Close"), "unique"); + ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "A.Close"), ""); ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "B.Close"), ""); - /* the doc of the method that lost the name is not attributed to the one - * that kept it: B.Close's [Target] is the edge, A.Close's a row */ + /* each method's doc is attributed to its own node: both [Target]s are + * edges, from the line of that method's doc, and neither is a row */ char props[512]; int n = 0; - dm_edge(db, "alpha.Close", "alpha.Target", props, sizeof(props), &n); + dm_edge(db, "alpha.A.Close", "alpha.Target", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"line\":9,\"count\":1")); + dm_edge(db, "alpha.B.Close", "alpha.Target", props, sizeof(props), &n); ASSERT_EQ(n, 1); ASSERT_NOT_NULL(strstr(props, "\"line\":12,\"count\":1")); - ASSERT_EQ(dm_count(db, "SELECT line FROM doc_link_unresolved WHERE rel_path = 'alpha/c.go' " - "AND raw = 'Target'"), - 9); - /* a type that lost its name to a method of that name is no target, the - * method is */ - ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "Open"), "graph_gap"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE rel_path = " + "'alpha/c.go' AND raw = 'Target'"), + 0); + /* a type and a method of the same name no longer share a node: [Open] + * is the type, [B.Open] the method */ + ASSERT_STR_EQ(go_reason(db, "alpha/c.go", "Open"), ""); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label <> 'Method' AND " + "qualified_name LIKE '%.alpha.Open'"), + 1); ASSERT_STR_EQ(go_tier(db, "alpha.UsesOpen", "alpha.Open"), "unique"); - ASSERT_EQ(dm_mentions_from(db, "alpha.UsesOpen"), 1); + ASSERT_STR_EQ(go_tier(db, "alpha.UsesOpen", "alpha.B.Open"), "unique"); + ASSERT_EQ(dm_mentions_from(db, "alpha.UsesOpen"), 2); /* build variants are one entity: whichever file holds the node */ static const go_want_edge_t variants[] = { @@ -1836,15 +1872,16 @@ TEST(doc_mentions_go_incremental_equals_full) { ASSERT_EQ(dm_index(repo, inc_db, NULL), 0); /* the preconditions the steps below move away from */ static const go_want_edge_t before[] = { - {"a.Use", "b.Target", "exact"}, {"a.Use", "b.Gone", "exact"}, - {"a.Use", "b.M", "exact"}, /* T.M, in the first file of the directory, keeps b.M */ - {"a.Use", "b.Name", "exact"}, /* W.Name, the later declaration, keeps b.Name */ - {"a.Use", "a.Local", "unique"}, {"a.doc.go.__file__", "a.Local", "unique"}, + {"a.Use", "b.Target", "exact"}, {"a.Use", "b.Gone", "exact"}, + {"a.Use", "b.T.M", "exact"}, /* #1913: every method has its own node */ + {"a.Use", "b.W.Name", "exact"}, {"a.Use", "b.V.Name", "exact"}, + {"b.U.M", "b.Target", "unique"}, /* U.M is a source of its own */ + {"a.Use", "a.Local", "unique"}, {"a.doc.go.__file__", "a.Local", "unique"}, {"a.Other", "b.Target", ""}, /* other.go does not import b */ }; ASSERT_EQ(go_bad_edges(inc_db, before, GO_COUNT(before)), 0); - ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.V.Name"), "graph_gap"); - ASSERT_STR_EQ(go_reason(inc_db, "b/c.go", "Target"), "graph_gap"); /* U.M has no node */ + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.V.Name"), ""); + ASSERT_STR_EQ(go_reason(inc_db, "b/c.go", "Target"), ""); /* a doc-comment edit re-extracts the file alone */ go_w(repo, "a/a.go", GO_INC_A_DOC_EDIT); @@ -1880,23 +1917,27 @@ TEST(doc_mentions_go_incremental_equals_full) { 0); ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.Gone"), "missing"); - /* the method that kept a shared node name goes: the other file's method of - * that name takes the node over, and its doc becomes a source */ + /* a method goes: links to it break, and another receiver's method of + * the same name, in another file, is untouched */ go_w(repo, "b/b.go", GO_INC_B_NO_M); ASSERT_EQ( dm_step(repo, inc_db, full_db, "name keeper removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.T.M"), "missing"); - ASSERT_STR_EQ(go_tier(inc_db, "b.M", "b.Target"), "unique"); + ASSERT_STR_EQ(go_tier(inc_db, "b.U.M", "b.Target"), "unique"); ASSERT_STR_EQ(go_reason(inc_db, "b/c.go", "Target"), ""); - /* declaration order decides which declaration keeps a shared name: - * swapping two methods moves the node, for files with no edge into it too */ + /* declaration order no longer decides anything: swapping two methods of + * the same name keeps both nodes and both links */ go_w(repo, "b/d.go", GO_INC_D_SWAPPED); - ASSERT_EQ(dm_step(repo, inc_db, full_db, "methods swapped", CBM_INCREMENTAL_ROUTE_FORCED_FULL), - 0); - ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.W.Name"), "graph_gap"); + /* and the scope blob does not change (no method w moves), so the step + * is an ordinary closure repair, no longer a forced full rebuild */ + ASSERT_EQ( + dm_step(repo, inc_db, full_db, "methods swapped", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.W.Name"), ""); ASSERT_STR_EQ(go_reason(inc_db, "a/a.go", "b.V.Name"), ""); + ASSERT_STR_EQ(go_tier(inc_db, "a.Use", "b.W.Name"), "exact"); + ASSERT_STR_EQ(go_tier(inc_db, "a.Use", "b.V.Name"), "exact"); /* renaming a target */ go_w(repo, "b/b.go", GO_INC_B_RENAME); @@ -2006,10 +2047,20 @@ TEST(doc_mentions_go_scope_delta_rules) { GO_DELTA_HEAD GO_DELTA_F GO_DELTA_G GO_DELTA_T GO_DELTA_M GO_DELTA_I GO_DELTA_C "var (\n\tV = 2\n)\n", CBM_DOCLINK_DELTA_GLOBAL, NULL}, - /* the declaration that keeps a shared node name changes */ - {"same-named declarations swapped", twins, - "package p\n\nfunc (T) Open() {}\nfunc Open() {}\n", CBM_DOCLINK_DELTA_GLOBAL, NULL}, - {"one of two same-named declarations removed", twins, "package p\n\nfunc Open() {}\n", + /* a method and a function of one name no longer share a node + * (#1913): swapping them changes no keeper */ + {"method and function swapped", twins, "package p\n\nfunc (T) Open() {}\nfunc Open() {}\n", + CBM_DOCLINK_DELTA_LOCAL, ""}, + /* the declaration that keeps a shared node name changes: two + * package-level declarations of one name still share . */ + {"same-named declarations swapped", "package p\n\nfunc Open() {}\ntype Open struct{}\n", + "package p\n\ntype Open struct{}\nfunc Open() {}\n", CBM_DOCLINK_DELTA_GLOBAL, NULL}, + /* the method of a method/function pair removed: a member left, the + * function keeps its node either way */ + {"method of a same-named pair removed", twins, "package p\n\nfunc Open() {}\n", + CBM_DOCLINK_DELTA_LOCAL, "Open"}, + {"one of two same-named declarations removed", + "package p\n\nfunc Open() {}\ntype Open struct{}\n", "package p\n\nfunc Open() {}\n", CBM_DOCLINK_DELTA_GLOBAL, NULL}, /* other files follow `type X = q.Y` through this file's imports */ {"import changed under a qualified alias", alias, @@ -2091,7 +2142,10 @@ TEST(doc_mentions_go_workers_agree) { ASSERT_STR_EQ(go_tier(par_db, "p00.T0", "p01.T0"), "exact"); ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "p01.Gone"), "missing"); ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "fmt.Println"), "external"); - ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "p01.T3.Close"), "graph_gap"); + /* #1913: [p01.T3.Close] lands on its own method node now; the graph_gap + * reason stays covered by Shared, one name declared by every file */ + ASSERT_STR_EQ(go_reason(par_db, "p00/f0.go", "p01.T3.Close"), ""); + ASSERT_STR_EQ(go_tier(par_db, "p00.T0", "p01.T3.Close"), "exact"); ASSERT_STR_EQ(go_reason(par_db, "p00/f3.go", "Shared"), "graph_gap"); ASSERT_TRUE(dm_count(par_db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS'") > 100); dm_unlink_db(par_db); From 5c5964efb9ab637cd6a42a12ff417f5ae0b502db Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 18 Oct 2026 23:28:43 +0200 Subject: [PATCH 11/19] fix(deploy): bound Helm render memory, no wrong image from a skipped change Review round 2 of #2677 (N1-N12). Every case ends in an explicit unknown, never in an OOM, a quadratic stall or a wrong image. Bounds (helm_template.c, pass_k8s.c): - N1: capture arrays grow by doubling, so n captures cost memory linear in n. Arena growth since the render began counts against a third budget (256 MiB) beside steps and output; a trip is logged with its reason. - N10: poisoned maps are a pointer hash set (merged values a path list), so a loop poisoning a new dict per iteration keeps reads O(1). Release images are deduplicated by hash. Wrong images (helm_template.c): - N2: a set / merge in a body an UNKNOWN condition skips is judged by its target whatever it is -- a call's result like `index .Values "image"` is evaluated in the frame: own, or a known map (poisoned), else it taints. - N3: putting a value another part holds into an own map revokes own; a part of a poisoned map is never own. - N4: a run-time set whose target is an UNKNOWN call result taints. - N5: an UNKNOWN break / continue spoils the rest of the loop body around it; a continue is a continue, not a break. - N6: the scan notes redefinitions and assignments, carries what it saw across the actions of one branch, and never judges a variable the part rebinds by its value in the frame. - N11: a define's "may not change" that leaned on a define still being scanned (mutual includes) is not kept. - N12: an include not run because its argument is poisoned or UNKNOWN spoils that argument (poisoned deep, or a taint). Releases and charts (pass_k8s.c, pass_terraform.c): - N8: an empty values.yaml is values {} (Helm's), not a file not read. - N9: a Terraform set_sensitive whose name overlaps a key an image reads, a Terraform postrender, or a kustomization with its own transformers (images, patches, replacements, transformers, components) leaves the release without an image of its own. Tests (N7): the symlink probes are POSIX-only (SKIP_PLATFORM on Windows). One test per item, RED on 4cea54f7 by its assertion; N10's is a correctness control, its cost measured (linear vs quadratic) outside the suite. Signed-off-by: Martin Vogel --- src/pipeline/helm_template.c | 583 ++++++++++++++++++++++--------- src/pipeline/helm_template.h | 1 + src/pipeline/pass_k8s.c | 148 +++++++- src/pipeline/pass_terraform.c | 123 ++++++- src/pipeline/pipeline_internal.h | 7 +- tests/test_grammar_probe_f.c | 88 ++++- tests/test_pipeline.c | 195 +++++++++++ 7 files changed, 953 insertions(+), 192 deletions(-) diff --git a/src/pipeline/helm_template.c b/src/pipeline/helm_template.c index fa516ae00..2f96db45d 100644 --- a/src/pipeline/helm_template.c +++ b/src/pipeline/helm_template.c @@ -39,13 +39,15 @@ enum { HELM_MAX_NESTING = 1000 }; /* Helm engine's recursionMaxNums */ enum { HELM_MAX_SET_INDEX = 65536 }; /* Helm strvals' MaxIndex for `--set a[n]` */ /* A render doing more than this -- steps (an action, an evaluation, a range - * iteration) or bytes written to any sink -- is one Helm never finishes + * iteration), bytes written to any sink, or memory it takes (the arena's + * growth: captures, list copies, strings) -- is one Helm never finishes * either (a range over a huge int, defines that call each other in a fan): * it stops, and every probe of the template is UNKNOWN. Counted, never timed; * far above any real chart (the 20 infra corpora and the helm oracle charts * never reach it). */ enum { HELM_MAX_STEPS = 10000000 }; -static const size_t HELM_MAX_OUTPUT = (size_t)CBM_SZ_64 * CBM_SZ_1K * CBM_SZ_1K; /* 64 MiB */ +static const size_t HELM_MAX_OUTPUT = (size_t)CBM_SZ_64 * CBM_SZ_1K * CBM_SZ_1K; /* 64 MiB */ +static const size_t HELM_MAX_MEMORY = (size_t)CBM_SZ_256 * CBM_SZ_1K * CBM_SZ_1K; /* 256 MiB */ /* ── Values: YAML -> JSON ─────────────────────────────────────────── */ @@ -1364,12 +1366,6 @@ typedef struct { hsink_t *out; /* NULL: the template's own output, which the probes read */ } hframe_t; -typedef struct { - const void *map; /* the hv_map_t / yyjson object changed */ - const char *path; /* its values path (NULL: none) */ - bool deep; /* merge: every map within it may have changed too */ -} hpoison_t; - typedef struct { CBMArena *a; yyjson_mut_doc *scratch; @@ -1382,14 +1378,16 @@ typedef struct { bool cont; long steps; size_t written; /* bytes appended to any sink */ - bool stop; /* over HELM_MAX_STEPS / HELM_MAX_OUTPUT: the render is UNKNOWN */ + size_t arena0; /* the arena's total when the render began */ + bool stop; /* over HELM_MAX_STEPS / _OUTPUT / _MEMORY: the render is UNKNOWN */ /* maps a set / unset / merge changed in place (not modelled): what they - * hold is UNKNOWN from then on */ - hpoison_t *poison; - int npoison; - int pcap; + * hold is UNKNOWN from then on -- by address ("%p" keys in the arena), + * and for a merge of values also by values path (everything below it) */ + CBMHashTable *poison_maps; + CBMHashTable *poison_paths; bool tainted; /* a change of unknown target: every capture ending later is UNKNOWN */ - CBMHashTable *mutates; /* define name -> it may change a map ((void *)1 / (void *)2) */ + CBMHashTable *mutates; /* define name -> it may change a map (he_define_may_mutate) */ + long busy_hits; /* defines met again while being scanned */ /* probes of the top-level template */ const char *top_src; size_t top_len; @@ -1408,9 +1406,14 @@ typedef struct { int tcap; } hst_t; +/* Is the render over its memory budget (the arena grown since it began)? */ +static bool hst_over_memory(const hst_t *st) { + return cbm_arena_total(st->a) - st->arena0 > HELM_MAX_MEMORY; +} + /* One step of work; false once the render is over its budget. */ static bool hst_step(hst_t *st) { - if (!st->stop && ++st->steps > HELM_MAX_STEPS) { + if (!st->stop && (++st->steps > HELM_MAX_STEPS || hst_over_memory(st))) { st->stop = true; } return !st->stop; @@ -1418,7 +1421,7 @@ static bool hst_step(hst_t *st) { /* n more bytes of output; false (and the render stopped) past the budget. */ static bool hst_write(hst_t *st, size_t n) { - if (st->stop || n > HELM_MAX_OUTPUT - st->written) { + if (st->stop || n > HELM_MAX_OUTPUT - st->written || hst_over_memory(st)) { st->stop = true; return false; } @@ -1548,20 +1551,54 @@ static bool hv_poisoned(const hst_t *st, const hv_t *v) { if (!map) { return false; } - for (int i = 0; i < st->npoison; i++) { - const hpoison_t *p = &st->poison[i]; - if (p->map == map) { - return true; - } - if (p->deep && p->path && v->path) { - size_t pl = strlen(p->path); - if (pl == 0 || (strncmp(v->path, p->path, pl) == 0 && - (v->path[pl] == '\0' || v->path[pl] == '.' || v->path[pl] == '['))) { - return true; - } - } + char key[CBM_SZ_32]; + snprintf(key, sizeof(key), "%p", map); + if (st->poison_maps && cbm_ht_get(st->poison_maps, key)) { + return true; } - return false; + if (!st->poison_paths || !v->path) { + return false; + } + /* a merged values path covers what is below it: each prefix of v's path + * that ends at a key ("", "a", "a.b", "a.b[0]" ...; an escaped `\.` is no + * end) is looked up */ + size_t pl = strlen(v->path); + char small[CBM_SZ_256]; + char *buf = pl < sizeof(small) ? small : cbm_alloc(CBM_MEM_CLASS_OTHER, pl + SKIP_ONE); + bool hit = false; + for (size_t i = 0; buf && i <= pl && !hit; i++) { + bool end = i == pl || ((v->path[i] == '.' || v->path[i] == '[') && + !(i > 0 && v->path[i - SKIP_ONE] == '\\')); + if (!end) { + continue; + } + memcpy(buf, v->path, i); + buf[i] = '\0'; + hit = cbm_ht_get(st->poison_paths, buf) != NULL; + } + if (buf != small) { + cbm_free(CBM_MEM_CLASS_OTHER, buf); + } + return hit || !buf; +} + +/* Key `k` into set *set (made when NULL; the key copied into the arena). */ +static bool hst_set_add(hst_t *st, CBMHashTable **set, const char *k) { + if (!*set) { + *set = cbm_ht_create(0); + } + if (!*set) { + return false; + } + if (cbm_ht_get(*set, k)) { + return true; + } + const char *dup = cbm_arena_strdup(st->a, k); + if (!dup) { + return false; + } + cbm_ht_set(*set, dup, (void *)(uintptr_t)SKIP_ONE); + return true; } static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth); @@ -1574,7 +1611,7 @@ static bool hst_poison(hst_t *st, const hv_t *v, bool deep) { } static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { - if (depth > HELM_MAX_NESTING) { + if (depth > HELM_MAX_NESTING || !hst_step(st)) { return false; } if (deep && v->k == HV_LIST) { @@ -1602,20 +1639,12 @@ static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { } } } - if (st->npoison == st->pcap) { - int cap = st->pcap ? st->pcap * PAIR_LEN : CBM_SZ_16; - hpoison_t *g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); - if (!g) { - return false; - } - if (st->npoison) { - memcpy(g, st->poison, (size_t)st->npoison * sizeof(*g)); - } - st->poison = g; - st->pcap = cap; + char key[CBM_SZ_32]; + snprintf(key, sizeof(key), "%p", map); + if (!hst_set_add(st, &st->poison_maps, key)) { + return false; } - st->poison[st->npoison++] = (hpoison_t){map, v->path, deep}; - return true; + return !(deep && v->path) || hst_set_add(st, &st->poison_paths, v->path); } static bool hn_is(TSNode n, const char *t) { @@ -1743,13 +1772,24 @@ static void hst_capture_end(hst_t *st) { return; } cbm_helm_probe_t *p = &st->probes[st->capture]; - const char **img = cbm_arena_alloc(st->a, (size_t)(p->n + SKIP_ONE) * sizeof(*img)); - const char **keys = cbm_arena_alloc(st->a, (size_t)(p->n + SKIP_ONE) * sizeof(*keys)); - if (img && keys) { - if (p->n) { - memcpy(img, p->image, (size_t)p->n * sizeof(*img)); - memcpy(keys, p->keys, (size_t)p->n * sizeof(*keys)); + if (p->n == p->cap) { + /* the captures of a probe grow by doubling: n captures cost O(n) */ + int cap = p->cap ? p->cap * PAIR_LEN : CBM_SZ_4; + const char **gi = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gi)); + const char **gk = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gk)); + if (gi && gk) { + if (p->n) { + memcpy(gi, p->image, (size_t)p->n * sizeof(*gi)); + memcpy(gk, p->keys, (size_t)p->n * sizeof(*gk)); + } + p->image = gi; + p->keys = gk; + p->cap = cap; } + } + const char **img = p->n < p->cap ? p->image : NULL; + const char **keys = p->n < p->cap ? p->keys : NULL; + if (img && keys) { img[p->n] = st->cap.unk || st->tainted ? NULL : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); @@ -1786,8 +1826,6 @@ static void hst_capture_end(hst_t *st) { if (!whole || (json && !keys[p->n])) { img[p->n] = NULL; /* a mark not written in full: no image, never one unmarked */ } - p->image = img; - p->keys = keys; p->n++; } st->capture = -SKIP_ONE; @@ -2453,14 +2491,25 @@ static bool he_deep_value(const hfresh_t *f, const char *src, TSNode x, int dept return true; } -/* Variable definition `def` (`$x := ...`) into f when what it makes is new. */ -static void he_fresh_note(hst_t *st, hfresh_t *f, const char *src, TSNode def) { - TSNode var = hn_field(def, "variable"); - TSNode val = hn_field(def, "value"); - hfresh_lvl_t lvl = ts_node_is_null(var) ? HF_NONE - : he_deep_value(f, src, val, 0) ? HF_DEEP - : he_fresh_value(f, src, val); - if (lvl == HF_NONE) { +/* Variable var is `lvl` new from here on (HF_NONE too: what it held before + * no longer counts -- the latest entry of a name is the one looked up). */ +/* Has the scan seen variable var defined or assigned (an entry in f)? */ +static bool he_fresh_seen(const hfresh_t *f, const char *src, TSNode var) { + size_t vl = 0; + const char *vs = he_var_text(src, var, &vl); + for (int i = 0; i < f->n; i++) { + if (f->len[i] == vl && memcmp(f->name[i], vs, vl) == 0) { + return true; + } + } + return false; +} + +/* Note in f that variable var is now lvl-new. A first HF_NONE entry is kept + * too: it hides the running frame's value of var from the scan. */ +static void he_fresh_put(hst_t *st, hfresh_t *f, const char *src, TSNode var, hfresh_lvl_t lvl) { + if (ts_node_is_null(var) || + (lvl == HF_NONE && he_fresh_seen(f, src, var) && he_fresh_var(f, src, var) == HF_NONE)) { return; } if (f->n == f->cap) { @@ -2483,6 +2532,77 @@ static void he_fresh_note(hst_t *st, hfresh_t *f, const char *src, TSNode def) { f->n++; } +/* Is expression x deep-new for the scan (f) -- or, a variable the scan did + * not see made, with fr (a part of the running template), a value of fr + * that is own? */ +static bool he_value_owned(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode x) { + if (he_deep_value(f, src, x, 0)) { + return true; + } + if (!fr || !hn_is(x, "variable") || he_fresh_seen(f, src, x)) { + return false; + } + size_t l = 0; + const char *name = hn_var_name(fr, x, &l); + hvar_t *v = hst_var(st, fr, name, l); + return v && v->v.own; +} + +/* Is call node c a set / unset / merge of variable var whose every other + * argument is own (the variable stays as new as it is)? */ +static bool he_self_change(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode c, + TSNode var) { + size_t fl = 0; + const char *fs = hn_is(c, "function_call") ? he_fn_text(src, c, &fl) : NULL; + TSNode args = fs && he_mutator(fs, fl) ? hn_field(c, "arguments") : (TSNode){0}; + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; + if (!hn_is(first, "variable")) { + return false; + } + size_t al = 0; + size_t bl = 0; + const char *as = he_var_text(src, first, &al); + const char *bs = he_var_text(src, var, &bl); + if (al != bl || memcmp(as, bs, al) != 0) { + return false; + } + for (uint32_t i = SKIP_ONE; i < na; i++) { + if (!he_value_owned(st, fr, f, src, ts_node_named_child(args, i))) { + return false; + } + } + return true; +} + +/* A definition (`$x := v`) or an assignment (`$x = v`) into f: how new $x is + * now -- what v is, or, for `$x = set $x ...` of own values, what $x was. */ +static void he_fresh_note(hst_t *st, hframe_t *fr, hfresh_t *f, const char *src, TSNode def) { + TSNode var = hn_field(def, "variable"); + TSNode val = hn_field(def, "value"); + if (ts_node_is_null(var)) { + return; + } + hfresh_lvl_t lvl = he_deep_value(f, src, val, 0) ? HF_DEEP : he_fresh_value(f, src, val); + if (lvl == HF_NONE && hn_is(def, "assignment") && he_self_change(st, fr, f, src, val, var)) { + lvl = he_fresh_var(f, src, var); + } + he_fresh_put(st, f, src, var, lvl); +} + +/* Are the values set / merge call c puts into its target own (see + * he_value_owned)? */ +static bool he_puts_deep(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode c) { + TSNode args = hn_field(c, "arguments"); + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + for (uint32_t i = SKIP_ONE; i < na; i++) { + if (!he_value_owned(st, fr, f, src, ts_node_named_child(args, i))) { + return false; + } + } + return !hn_is(ts_node_parent(c), "chained_pipeline"); /* a piped value: not known here */ +} + /* Does expression x read `.` (a field of it)? In a with / range body `.` is * not the frame's. */ static bool he_reads_dot(TSNode x) { @@ -2505,12 +2625,36 @@ static bool he_reads_dot(TSNode x) { return found; } +/* Does expression x name a variable the scan (f) saw defined or assigned -- + * one whose value in the running frame is not the one x would read? */ +static bool he_names_rebound(const hfresh_t *f, const char *src, TSNode x) { + TSTreeCursor cur = ts_tree_cursor_new(x); + bool found = false; + for (bool more = true; more && !found;) { + TSNode c = ts_tree_cursor_current_node(&cur); + found = hn_is(c, "variable") && he_fresh_seen(f, src, c); + if (found || ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return found; +} + /* What a change of the value of expression x (in frame fr; NULL: not * running) would reach: false when it reaches nothing another part holds -- * x is own, or a known map, now poisoned (`deep`: a merge). A value not known - * here: true. */ -static bool he_change_reaches(hst_t *st, hframe_t *fr, TSNode x, bool dot_known, bool deep) { - if (!fr || ts_node_is_null(x) || (!dot_known && he_reads_dot(x))) { + * here -- or x naming a variable the part (f) redefined: true. */ +static bool he_change_reaches(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode x, + bool dot_known, bool deep) { + if (!fr || ts_node_is_null(x) || (!dot_known && he_reads_dot(x)) || + he_names_rebound(f, src, x)) { return true; } /* read without recording: the part is not run, its reads are no capture's */ @@ -2523,21 +2667,27 @@ static bool he_change_reaches(hst_t *st, hframe_t *fr, TSNode x, bool dot_known, return !poisoned && !v.own; } +static void he_revoke_own(hst_t *st, hframe_t *fr, TSNode x); + /* May running node n (of text src) change a map another part holds -- a set / - * unset / merge whose target is `.`, `$`, a field path, or a variable n did - * not make itself from a new map (`$d := dict ...`), in n or in a define it - * includes (whose changes reach only its argument)? A call's result as the - * target is a new value: no. With fr (n is a part of the running template - * not run), a target or include argument that is a value of fr is looked at: - * own, or a known map (then poisoned): no. An include of a computed name is a - * template file by path, another template's work, which no render models. - * The text a tpl renders is not looked into. */ + * unset / merge whose target is not a value n made new itself (`$d := dict`, + * `set (dict) ...`; a merge needs it deep-new), in n or in a define it + * includes (whose changes reach only its argument)? With fr (n is a part of + * the running template not run), a target or include argument -- `.`, a + * field, a call's result such as `index .Values "image"`, a variable of fr -- + * is looked at: own, or a known map (then poisoned): no. A change that puts a + * value another part may hold into a variable makes it no longer new / own. + * An include of a computed name is a template file by path, another + * template's work, which no render models. The text a tpl renders is not + * looked into. `made` (NULL: none) is what the parts before n in its body + * made new or redefined; n's own are added to it. */ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hframe_t *fr, - bool dot_known) { + bool dot_known, hfresh_t *made) { if (depth > HELM_MAX_NESTING) { return true; } - hfresh_t fresh = {0}; /* variables n makes from a new map */ + hfresh_t own = {0}; /* variables n makes from a new map */ + hfresh_t *fresh = made ? made : &own; TSTreeCursor cur = ts_tree_cursor_new(n); bool found = false; bool more = true; @@ -2546,20 +2696,31 @@ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hfram const char *t = ts_node_type(c); size_t fl = 0; const char *fs = NULL; - if (strcmp(t, "variable_definition") == 0) { - he_fresh_note(st, &fresh, src, c); + if (strcmp(t, "variable_definition") == 0 || strcmp(t, "assignment") == 0) { + he_fresh_note(st, fr, fresh, src, c); } else if (strcmp(t, "function_call") == 0 && (fs = he_fn_text(src, c, &fl)) != NULL && he_mutator(fs, fl)) { TSNode args = hn_field(c, "arguments"); TSNode target = ts_node_is_null(args) ? (TSNode){0} : ts_node_named_child(args, 0); bool deep = fs[0] == 'm'; /* a merge reaches the maps within its target too */ - if (hn_is(target, "variable")) { - hfresh_lvl_t lvl = he_fresh_var(&fresh, src, target); - bool spared = lvl == HF_DEEP || (lvl == HF_SHALLOW && !deep); - found = !spared && he_change_reaches(st, fr, target, dot_known, deep); - } else if (ts_node_is_null(target) || hn_is(target, "dot") || hn_is(target, "field") || - hn_is(target, "selector_expression")) { - found = he_change_reaches(st, fr, target, dot_known, deep); + bool puts_deep = he_puts_deep(st, fr, fresh, src, c); + /* how new the target is: a variable n made, or a new value itself + * (`set (dict) ...`); anything else -- `.`, a field, a call's + * result (`index .Values "image"`) -- is looked at in fr */ + hfresh_lvl_t lvl = hn_is(target, "variable") ? he_fresh_var(fresh, src, target) + : ts_node_is_null(target) ? HF_NONE + : he_deep_value(fresh, src, target, 0) + ? HF_DEEP + : he_fresh_value(fresh, src, target); + bool spared = lvl == HF_DEEP || (lvl == HF_SHALLOW && !deep); + found = !spared && he_change_reaches(st, fr, fresh, src, target, dot_known, deep); + if (!puts_deep && hn_is(target, "variable")) { + /* it now holds a value another part may hold */ + if (lvl == HF_DEEP) { + he_fresh_put(st, fresh, src, target, HF_SHALLOW); + } else if (fr && lvl == HF_NONE) { + he_revoke_own(st, fr, target); + } } } else if (strcmp(t, "function_call") == 0 || strcmp(t, "template_action") == 0 || strcmp(t, "block_action") == 0) { @@ -2572,8 +2733,8 @@ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hfram ? (ts_node_is_null(args) ? (TSNode){0} : ts_node_named_child(args, SKIP_ONE)) : hn_field(c, "argument"); - found = !he_deep_value(&fresh, src, arg, 0) && - he_change_reaches(st, fr, arg, dot_known, true); + found = !he_deep_value(fresh, src, arg, 0) && + he_change_reaches(st, fr, fresh, src, arg, dot_known, true); } } else if (strcmp(t, "identifier") == 0 && (fs = he_fn_text(src, c, &fl)) != NULL && he_mutator(fs, fl) && hn_is(ts_node_parent(c), "chained_pipeline")) { @@ -2594,9 +2755,12 @@ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hfram return found || st->stop; } -/* Memoized per define (a define reached again while it is scanned: no). */ +/* Memoized per define. A define reached again while it is being scanned + * counts as no for now; a no that leaned on such a one is not kept (STALE: + * scanned again when next asked), so defines including each other never fix + * a wrong no -- a yes is final. */ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { - enum { HM_NO = 1, HM_YES = 2 }; + enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3, HM_STALE = 4 }; const hd_def_t *d = hd_find(st->defs, name); if (!d) { return false; /* a define not found is an error: nothing of it runs */ @@ -2604,36 +2768,54 @@ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { if (!st->mutates) { st->mutates = cbm_ht_create(0); } - void *memo = st->mutates ? cbm_ht_get(st->mutates, name) : NULL; - if (memo) { - return memo == (void *)(uintptr_t)HM_YES; + uintptr_t memo = st->mutates ? (uintptr_t)cbm_ht_get(st->mutates, name) : 0; + if (memo == HM_YES || memo == HM_NO) { + return memo == HM_YES; } - const char *key = cbm_arena_strdup(st->a, name); - if (st->mutates && key) { - cbm_ht_set(st->mutates, key, (void *)(uintptr_t)HM_NO); + if (memo == HM_BUSY) { + st->busy_hits++; + return false; } - bool m = he_may_mutate(st, d->file->src, d->node, depth, NULL, false); + const char *key = memo ? cbm_ht_get_key(st->mutates, name) : cbm_arena_strdup(st->a, name); if (st->mutates && key) { - cbm_ht_set(st->mutates, key, (void *)(uintptr_t)(m ? HM_YES : HM_NO)); + cbm_ht_set(st->mutates, key, (void *)(uintptr_t)HM_BUSY); + } + long hits = st->busy_hits; + bool m = he_may_mutate(st, d->file->src, d->node, depth, NULL, false, NULL); + bool leaned = st->busy_hits > hits; /* on a define still being scanned */ + if (st->mutates && key) { + uintptr_t res = m ? HM_YES : leaned ? HM_STALE : HM_NO; + cbm_ht_set(st->mutates, key, (void *)res); } return m; } static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n); +/* A define not run would change its argument v (or a map within it): a known + * one is poisoned whole; one not known that is not own spoils what follows. */ +static void he_spoil_arg(hst_t *st, const hv_t *v) { + bool known = hv_is_map(v) || hv_is_list(v); + if (!(known && hst_poison(st, v, true)) && !v->own && + (v->k == HV_UNK || hv_is_map(v) || hv_is_list(v))) { + st->tainted = true; + } +} + /* Is the result of `fn` over a own (see hv_t): a part dug / got / indexed / - * taken out of an own value; a dict / list / default / coalesce / ternary - * whose every map or list (and every value not known) is own? */ -static bool he_own_result(const char *fn, const hv_t *a, int n) { + * taken out of an own value no change has touched (a poisoned map may hold + * another's now); a dict / list / default / coalesce / ternary whose every map + * or list (and every value not known) is own? */ +static bool he_own_result(const hst_t *st, const char *fn, const hv_t *a, int n) { if (n < SKIP_ONE) { return strcmp(fn, "dict") == 0 || strcmp(fn, "list") == 0; /* empty and new */ } if (strcmp(fn, "dig") == 0) { - return a[n - SKIP_ONE].own; + return a[n - SKIP_ONE].own && !hv_poisoned(st, &a[n - SKIP_ONE]); } if (strcmp(fn, "get") == 0 || strcmp(fn, "index") == 0 || strcmp(fn, "first") == 0 || strcmp(fn, "last") == 0) { - return a[0].own; + return a[0].own && !hv_poisoned(st, &a[0]); } if (strcmp(fn, "dict") == 0 || strcmp(fn, "list") == 0 || strcmp(fn, "default") == 0 || strcmp(fn, "coalesce") == 0 || strcmp(fn, "ternary") == 0) { @@ -2654,28 +2836,73 @@ static hv_t he_apply(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { return HV_UNKNOWN_OWN; /* fromYaml / deepCopy: a new value, known or not */ } hv_t r = he_apply_fn(st, fr, fn, a, n); - r.own = r.own || he_own_result(fn, a, n); + r.own = r.own || he_own_result(st, fn, a, n); return r; } +/* The variable expression x reads its value out of (`$v`, `$v.a`, `(index $v + * "a")`, `get $v "a"`, `dig "a" $v`); a null node when there is none. */ +static TSNode he_root_var(const hframe_t *fr, TSNode x) { + for (int depth = 0; depth < HELM_MAX_NESTING && !ts_node_is_null(x); depth++) { + if (hn_is(x, "variable")) { + return x; + } + if (hn_is(x, "parenthesized_pipeline") && ts_node_named_child_count(x) == SKIP_ONE) { + x = ts_node_named_child(x, 0); + } else if (hn_is(x, "selector_expression")) { + x = hn_field(x, "operand"); + } else if (hn_is(x, "function_call")) { + size_t fl = 0; + const char *fs = he_fn_text(fr->src, x, &fl); + TSNode args = hn_field(x, "arguments"); + uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + bool last = fs && fl == strlen("dig") && memcmp(fs, "dig", fl) == 0; + bool first = fs && ((fl == strlen("get") && memcmp(fs, "get", fl) == 0) || + (fl == strlen("index") && memcmp(fs, "index", fl) == 0) || + (fl == strlen("first") && memcmp(fs, "first", fl) == 0) || + (fl == strlen("last") && memcmp(fs, "last", fl) == 0)); + if (!na || (!first && !last)) { + break; + } + x = ts_node_named_child(args, last ? na - SKIP_ONE : 0); + } else { + break; + } + } + return (TSNode){0}; +} + +/* A change put a value another part may hold into what expression x reads: + * the variable it reads it out of is own no more. */ +static void he_revoke_own(hst_t *st, hframe_t *fr, TSNode x) { + TSNode var = he_root_var(fr, x); + if (ts_node_is_null(var)) { + return; + } + size_t l = 0; + const char *name = hn_var_name(fr, var, &l); + hvar_t *v = hst_var(st, fr, name, l); + if (v) { + v->v.own = false; + } +} + static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { for (int i = 0; i < n; i++) { - if (a[i].k == HV_UNK) { - /* an include of a known define not run: a change it would have - * made to its argument is missed, unless that one is own (one of a - * name not known -- a template file by path -- is another - * template's work, which no render models) */ - if (strcmp(fn, "include") == 0 && a[0].k == HV_STR && n == PAIR_LEN && !a[1].own) { + if (a[i].k == HV_UNK || hv_poisoned(st, &a[i])) { + /* an include of a known define not run (an argument UNKNOWN or + * poisoned): a change it would make to its argument is missed -- + * a known argument is poisoned whole, one not known and not own + * spoils what follows (one of a name not known -- a template file + * by path -- is another template's work, which no render models) */ + if (strcmp(fn, "include") == 0 && a[0].k == HV_STR && n == PAIR_LEN) { const char *dn = cbm_arena_strndup(st->a, a[0].s, a[0].sn); if (!dn || he_define_may_mutate(st, dn, st->nesting)) { - st->tainted = true; + he_spoil_arg(st, &a[1]); } } return HV_UNKNOWN; } - if (hv_poisoned(st, &a[i])) { - return HV_UNKNOWN; - } } if (strcmp(fn, "include") == 0 && n == PAIR_LEN && a[0].k == HV_STR) { hsink_t out = {0}; @@ -3031,8 +3258,8 @@ static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { if (he_mutator(fn, fl)) { /* set / unset / merge change a map in place -- not modelled: what it * holds is UNKNOWN from here on (after a merge, every map within it). - * A target not known that may be one another part holds -- a variable, - * `.`, a field path; a call's result is a new value -- spoils every + * A target not known that is not own -- whatever expression gives it: + * index / get / first return a map another part holds -- spoils every * image after it. */ TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; hv_t target = na ? he_eval(st, fr, first) : piped ? *piped : HV_UNKNOWN; @@ -3042,13 +3269,14 @@ static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { hv_t v = i < na ? he_eval(st, fr, ts_node_named_child(args, i)) : *piped; put_own = put_own && (v.own || (v.k != HV_UNK && !hv_is_map(&v) && !hv_is_list(&v))); } - bool held = ts_node_is_null(first) || hn_is(first, "variable") || hn_is(first, "dot") || - hn_is(first, "field") || hn_is(first, "selector_expression"); /* a known map is poisoned even when own: what it holds is not known now */ bool poisoned = hst_poison(st, &target, fn[0] == 'm'); - if (!poisoned && !target.own && (held || hv_is_map(&target))) { + if (!poisoned && !target.own) { st->tainted = true; } + if (!put_own && !ts_node_is_null(first)) { + he_revoke_own(st, fr, first); + } /* the changed map is what set / merge return */ return target.own && put_own ? HV_UNKNOWN_OWN : HV_UNKNOWN; } @@ -3235,21 +3463,6 @@ static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field) } } -/* Is expression x deep-new (f: what the scan saw made new), or a variable of - * frame fr whose value is own? */ -static bool he_part_own(hst_t *st, hframe_t *fr, const hfresh_t *f, TSNode x) { - if (he_deep_value(f, fr->src, x, 0)) { - return true; - } - if (!hn_is(x, "variable")) { - return false; - } - size_t l = 0; - const char *name = hn_var_name(fr, x, &l); - hvar_t *v = hst_var(st, fr, name, l); - return v && v->v.own; -} - /* Does assigning expression `value` to variable var keep it own (when it is): * a deep-new value, or a set / merge of var itself with own values * (`$dst = ... | fromYaml | merge $dst`)? */ @@ -3287,36 +3500,40 @@ static bool he_assign_owns(hst_t *st, hframe_t *fr, const hfresh_t *f, TSNode va return false; } for (uint32_t i = SKIP_ONE; i < na; i++) { - if (!he_part_own(st, fr, f, ts_node_named_child(args, i))) { + if (!he_value_owned(st, fr, f, src, ts_node_named_child(args, i))) { return false; } } - return ts_node_is_null(piped) || he_part_own(st, fr, f, piped); + return ts_node_is_null(piped) || he_value_owned(st, fr, f, src, piped); } +/* How a part not run would leave the loop around it. */ +enum { HL_NONE = 0, HL_CONTINUE = 1, HL_BREAK = 2 }; + /* The variables of the frames around n that n assigns (`$x = ...`) become - * UNKNOWN (an own one stays own when what n gives it keeps it so); true when - * n would leave the loop around it -- a break / continue not inside a range - * of n's own (`in_range`: n is a range's body). */ -static bool he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { + * UNKNOWN (an own one stays own when what n gives it keeps it so); returns + * how n would leave the loop around it (HL_*) -- by a break / continue not + * inside a range of n's own (`in_range`: n is a range's body). `fresh` is + * what the parts before n in its body made new; n's own are added to it. */ +static int he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range, hfresh_t *fresh) { TSTreeCursor cur = ts_tree_cursor_new(n); - hfresh_t fresh = {0}; /* variables n makes new */ - bool leave = false; + int leave = HL_NONE; bool more = true; while (more && hst_step(st)) { TSNode c = ts_tree_cursor_current_node(&cur); const char *t = ts_node_type(c); if (strcmp(t, "variable_definition") == 0) { - he_fresh_note(st, &fresh, fr->src, c); + he_fresh_note(st, fr, fresh, fr->src, c); } else if (strcmp(t, "assignment") == 0) { TSNode var = hn_field(c, "variable"); size_t l = 0; const char *name = ts_node_is_null(var) ? NULL : hn_var_name(fr, var, &l); hvar_t *v = name ? hst_var(st, fr, name, l) : NULL; if (v) { - bool keeps = v->v.own && he_assign_owns(st, fr, &fresh, var, hn_field(c, "value")); + bool keeps = v->v.own && he_assign_owns(st, fr, fresh, var, hn_field(c, "value")); v->v = keeps ? HV_UNKNOWN_OWN : HV_UNKNOWN; } + he_fresh_note(st, fr, fresh, fr->src, c); } else if (strcmp(t, "break_action") == 0 || strcmp(t, "continue_action") == 0) { /* a range around it inside n (n itself included) is the one it leaves */ bool own = in_range || hn_is(n, "range_action"); @@ -3324,7 +3541,9 @@ static bool he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { p = ts_node_parent(p); own = hn_is(p, "range_action"); } - leave = leave || !own; + if (!own) { + leave |= t[0] == 'b' ? HL_BREAK : HL_CONTINUE; + } } if (ts_tree_cursor_goto_first_child(&cur)) { continue; @@ -3340,32 +3559,70 @@ static bool he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { return leave; } -/* The bodies of if / with / range n, not run because its condition or - * collection is UNKNOWN (that one was run): their output, the variables they - * would assign and a change they would make are UNKNOWN; a loop they would - * leave is left now (its later iterations are not known). */ -static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { - hst_out_unknown(st, fr); - bool range = hn_is(n, "range_action"); - bool with = hn_is(n, "with_action"); - bool leave = false; +/* The children of n but its condition / collection (what was run), each: + * the variables it would assign and the changes it would make spoiled (see + * he_scan_skipped / he_may_mutate); returns how it would leave the loop. */ +static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, const char *only, bool in_range, + bool dot_known) { + int leave = HL_NONE; bool mutates = false; + /* what a branch's parts made new so far: a later part of it sees it */ + hfresh_t scanned = {0}; + hfresh_t made = {0}; + const char *branch = NULL; uint32_t cc = ts_node_child_count(n); for (uint32_t i = 0; i < cc; i++) { TSNode c = ts_node_child(n, i); const char *f = ts_node_field_name_for_child(n, i); + if (f && strcmp(f, "condition") == 0) { + branch = NULL; /* an else if: a new branch */ + } if (!ts_node_is_named(c) || hn_is(c, "range_variable_definition") || - (f && (strcmp(f, "condition") == 0 || strcmp(f, "range") == 0))) { + (f && (strcmp(f, "condition") == 0 || strcmp(f, "range") == 0)) || + (only && !(f && strcmp(f, only) == 0))) { continue; } - leave = he_scan_skipped(st, fr, c, range) || leave; - mutates = mutates || he_may_mutate(st, fr->src, c, st->nesting, fr, !range && !with); + if (f && (!branch || strcmp(f, branch) != 0)) { + branch = f; + scanned = (hfresh_t){0}; + made = (hfresh_t){0}; + } + leave |= he_scan_skipped(st, fr, c, in_range, &scanned); + mutates = mutates || he_may_mutate(st, fr->src, c, st->nesting, fr, dot_known, &made); } if (mutates) { st->tainted = true; } - if (leave) { + return leave; +} + +/* The bodies of if / with / range n, not run because its condition or + * collection is UNKNOWN (that one was run): their output, the variables they + * would assign and a change they would make are UNKNOWN. A break / continue + * in them may or may not leave the loop around: what the rest of that loop's + * body would assign or change is UNKNOWN too, and a break leaves it now (its + * later iterations are not known), a continue this iteration. */ +static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { + hst_out_unknown(st, fr); + bool range = hn_is(n, "range_action"); + bool with = hn_is(n, "with_action"); + int leave = he_spoil_parts(st, fr, n, NULL, range, !range && !with); + if (leave == HL_NONE) { + return; + } + TSNode loop = ts_node_parent(n); + for (int depth = 0; + !ts_node_is_null(loop) && !hn_is(loop, "range_action") && depth < HELM_MAX_NESTING; + depth++) { + loop = ts_node_parent(loop); + } + if (hn_is(loop, "range_action")) { + (void)he_spoil_parts(st, fr, loop, "body", true, false); + } + if (leave & HL_BREAK) { st->brk = true; + } else { + st->cont = true; } } @@ -3508,9 +3765,9 @@ static void he_template(hst_t *st, hframe_t *fr, TSNode n) { if (a.k == HV_UNK || hv_poisoned(st, &a)) { hst_out_unknown(st, fr); /* the define not run: a change it would make to its argument is - * missed, unless that one is own */ - if (!a.own && he_define_may_mutate(st, name, st->nesting)) { - st->tainted = true; + * missed (see he_spoil_arg) */ + if (he_define_may_mutate(st, name, st->nesting)) { + he_spoil_arg(st, &a); } return; } @@ -3612,6 +3869,7 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, probes[i].n = 0; probes[i].image = NULL; probes[i].keys = NULL; + probes[i].cap = 0; } if (!src || nprobes == 0) { return true; @@ -3622,7 +3880,8 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, .top_len = len, .probes = probes, .nprobes = nprobes, - .capture = -SKIP_ONE}; + .capture = -SKIP_ONE, + .arena0 = cbm_arena_total(a)}; st.scratch = yyjson_mut_doc_new(cbm_json_writer_allocator()); st.parser = ts_parser_new(); if (!st.scratch || !st.parser || @@ -3679,6 +3938,12 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, if (st.mutates) { cbm_ht_free(st.mutates); } + if (st.poison_maps) { + cbm_ht_free(st.poison_maps); + } + if (st.poison_paths) { + cbm_ht_free(st.poison_paths); + } if (st.stop) { /* Helm would not finish this template: none of its images is known, * also those captured before the budget ran out */ @@ -3689,7 +3954,9 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, } } cbm_log_warn("helm.render.budget", "file", rel ? rel : "", "reason", - st.steps > HELM_MAX_STEPS ? "steps" : "output"); + st.steps > HELM_MAX_STEPS ? "steps" + : st.written >= HELM_MAX_OUTPUT ? "output" + : "memory"); } return !st.stop; } diff --git a/src/pipeline/helm_template.h b/src/pipeline/helm_template.h index 47bdb007b..44985a245 100644 --- a/src/pipeline/helm_template.h +++ b/src/pipeline/helm_template.h @@ -68,6 +68,7 @@ void cbm_helm_defines_free(cbm_helm_defines_t *d); typedef struct { int line; /* 1-based line of the key in the template */ int n; /* captures */ + int cap; /* room in image / keys (the render's own) */ const char **image; /* the rendered scalar, YAML quoting removed; NULL: UNKNOWN */ const char **keys; /* JSON array of the values paths it read; NULL: none */ } cbm_helm_probe_t; diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 9a945c1d5..d20ca4cce 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -3904,6 +3904,7 @@ typedef struct { k8s_himg_t *v; int n; int cap; + CBMHashTable *seen; /* "src:img" of each v (keys cbm_alloc'd): one look-up a push */ } k8s_himgs_t; typedef struct { @@ -3919,6 +3920,9 @@ typedef struct { int tcap; k8s_himgs_t dflt; /* the containers' images with the defaults */ int edges; + /* the release rendered: its set_sensitive names (JSON array; NULL: none) -- + * an image reading one is not known, its value never read */ + const char *sensitive; } k8s_helm_t; /* The templated images of the facts, kept past them. */ @@ -3948,16 +3952,29 @@ static char *k8s_repo_read(cbm_pipeline_ctx_t *ctx, const char *rel, int *len) { } char path[CBM_SZ_4K]; int n = snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); - return n > 0 && (size_t)n < sizeof(path) ? k8s_read_file(path, len) : NULL; + if (n <= 0 || (size_t)n >= sizeof(path)) { + return NULL; + } + char *buf = k8s_read_file(path, len); + cbm_path_info_t info; + if (!buf && cbm_path_info_utf8(path, &info) == CBM_PATH_INFO_OK && info.is_regular && + !info.is_symlink && info.size == 0) { + /* an empty file is read: empty text (padded like every read) */ + enum { CBM_TS_EMPTY_PAD = 16 }; + buf = cbm_calloc(CBM_MEM_CLASS_OTHER, CBM_TS_EMPTY_PAD); + *len = 0; + } + return buf; } -/* Is repository path `rel` there at all -- a file, a symlink, anything (a - * link is reported, never followed)? */ +/* Is repository path `rel` there, or can that not be told (a link is + * reported, never followed; only a path proven absent is not there)? */ static bool k8s_repo_has(cbm_pipeline_ctx_t *ctx, const char *rel) { char path[CBM_SZ_4K]; int n = snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); cbm_path_info_t info; - return n > 0 && (size_t)n < sizeof(path) && cbm_path_info_utf8(path, &info) == 0; + return n > 0 && (size_t)n < sizeof(path) && + cbm_path_info_utf8(path, &info) != CBM_PATH_INFO_ABSENT; } /* A file of `dir` (one of `names`), parsed as values; *found when one exists. @@ -4232,14 +4249,38 @@ static const char *k8s_helm_overridden(k8s_helm_t *h, const char *keys, yyjson_m } static void k8s_himg_push(k8s_himgs_t *l, k8s_himg_t x) { - for (int i = 0; i < l->n; i++) { - if (l->v[i].src == x.src && l->v[i].img == x.img) { - return; /* the first (in file order) of the same Image stands for it */ - } + char key[CBM_SZ_64]; + snprintf(key, sizeof(key), "%lld:%lld", (long long)x.src, (long long)x.img); + if (!l->seen) { + l->seen = cbm_ht_create(0); } - if (k8s_grow((void **)&l->v, &l->cap, l->n, sizeof(*l->v))) { - l->v[l->n++] = x; + if (!l->seen || cbm_ht_get(l->seen, key)) { + return; /* the first (in file order) of the same Image stands for it */ } + size_t kl = strlen(key) + SKIP_ONE; + char *k = cbm_alloc(CBM_MEM_CLASS_OTHER, kl); + if (!k || !k8s_grow((void **)&l->v, &l->cap, l->n, sizeof(*l->v))) { + cbm_free(CBM_MEM_CLASS_OTHER, k); + return; + } + memcpy(k, key, kl); + cbm_ht_set(l->seen, k, (void *)(uintptr_t)SKIP_ONE); + l->v[l->n++] = x; +} + +static void k8s_himg_key_free(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void k8s_himgs_free(k8s_himgs_t *l) { + if (l->seen) { + cbm_ht_foreach(l->seen, k8s_himg_key_free, NULL); + cbm_ht_free(l->seen); + } + cbm_free(CBM_MEM_CLASS_OTHER, l->v); + *l = (k8s_himgs_t){0}; } /* Does `src` USE Image `img` already by a literal reference? */ @@ -4313,8 +4354,68 @@ static void k8s_helm_emit_uses(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const k8s } } +/* Do values paths a and b overlap -- one is the other or lies within it + * (a key boundary `.` / `[` after the shorter; "" is the whole values)? */ +static bool k8s_path_overlap(const char *a, const char *b) { + size_t al = strlen(a); + size_t bl = strlen(b); + size_t m = al < bl ? al : bl; + if (strncmp(a, b, m) != 0) { + return false; + } + char c = (al < bl ? b : a)[m]; + return m == 0 || c == '\0' || c == '.' || c == '['; +} + +/* Does values path `path` (of the release's chart) overlap a set_sensitive + * name of the release rendered? */ +static bool k8s_helm_sensitive_path(const k8s_helm_t *h, const char *path) { + if (!h->sensitive || !path) { + return false; + } + yyjson_doc *d = yyjson_read(h->sensitive, strlen(h->sensitive), 0); + bool hit = !d; /* names not read: say it touches */ + size_t idx, max; + yyjson_val *k; + yyjson_arr_foreach(d ? yyjson_doc_get_root(d) : NULL, idx, max, k) { + const char *s = yyjson_get_str(k); + hit = hit || (s && k8s_path_overlap(path, s)); + } + if (d) { + yyjson_doc_free(d); + } + return hit; +} + +/* Does an image reading values keys `keys` (a JSON array, of its chart at + * key path `prefix` within the release's; global.* are the release's own) + * read a set_sensitive name of the release rendered? */ +static bool k8s_helm_sensitive_keys(k8s_helm_t *h, const char *keys, const char *prefix) { + if (!h->sensitive || !keys) { + return false; + } + yyjson_doc *d = yyjson_read(keys, strlen(keys), 0); + bool hit = !d; + size_t idx, max; + yyjson_val *k; + yyjson_arr_foreach(d ? yyjson_doc_get_root(d) : NULL, idx, max, k) { + const char *key = yyjson_get_str(k); + bool global = key && strncmp(key, "global.", strlen("global.")) == 0; + const char *full = !key ? NULL + : global || !prefix || !prefix[0] + ? key + : cbm_arena_sprintf(&h->a, "%s.%s", prefix, key); + hit = hit || !full || k8s_helm_sensitive_path(h, full); + } + if (d) { + yyjson_doc_free(d); + } + return hit; +} + /* Chart `from` -DEPLOYS {via, ref: image, image, override_of}-> the images an - * overlay changed: rendered there, absent from the defaults' images. */ + * overlay changed: rendered there, absent from the defaults' images. An image + * reading a set_sensitive name (whose value is not read) is not known. */ static void k8s_helm_emit_overrides(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t from, const char *via, const k8s_himgs_t *dflt, const k8s_himgs_t *got, yyjson_mut_val *overlay, @@ -4328,7 +4429,7 @@ static void k8s_helm_emit_overrides(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int6 strcmp(dflt->v[j].ref, x->ref) == 0; } const char *ovr = k8s_helm_overridden(h, x->keys, overlay, globals, prefix); - if (same || !ovr || from <= 0) { + if (same || !ovr || from <= 0 || k8s_helm_sensitive_keys(h, x->keys, prefix)) { continue; } k8s_himg_t e = {from, x->img, x->ref, NULL, ovr}; @@ -4593,7 +4694,7 @@ static void k8s_helm_release_local(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 yyjson_mut_val *globals = yyjson_mut_obj_get(rv, "global"); k8s_helm_emit_overrides(ctx, h, src, via, &h->dflt, &got, overlay, globals, path && path[0] ? path : NULL, emitted); - cbm_free(CBM_MEM_CLASS_OTHER, got.v); + k8s_himgs_free(&got); } cbm_free(CBM_MEM_CLASS_OTHER, srcs); from = to; @@ -4647,7 +4748,9 @@ static void k8s_helm_convention(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t : "", dig[0] ? dig : tag) : NULL; - int64_t img = ref && k8s_helm_has_host(ref) && cbm_image_ref_valid(ref) + /* a set_sensitive name within the mapping: its value is not read */ + bool sensitive = k8s_helm_sensitive_path(h, path ? path : ""); + int64_t img = ref && !sensitive && k8s_helm_has_host(ref) && cbm_image_ref_valid(ref) ? cbm_pipeline_image_node(ctx, ref) : 0; int before = emitted->n; @@ -4699,7 +4802,7 @@ static void k8s_helm_release(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8s_helm cbm_ht_get(b->charts, dir)) { k8s_helm_release_local(ctx, b, h, l, src, via, dir, rv, &emitted); } - cbm_free(CBM_MEM_CLASS_OTHER, emitted.v); + k8s_himgs_free(&emitted); } /* A node's document (lines start_line..end_line of its file) as values. */ @@ -4792,6 +4895,16 @@ static yyjson_mut_val *k8s_helm_kustomize_values(cbm_pipeline_ctx_t *ctx, k8s_he char *src = kust->file_path ? k8s_repo_read(ctx, kust->file_path, &len) : NULL; yyjson_mut_val *doc = src ? cbm_helm_values_parse(h->doc, src, (size_t)len) : NULL; cbm_free(CBM_MEM_CLASS_OTHER, src); + /* kustomize runs the kustomization's own transformers over the inflated + * chart: images / patches / replacements may change what it deploys */ + static const char *const TRANSFORMS[] = { + "images", "patches", "patchesJson6902", "patchesStrategicMerge", + "replacements", "transformers", "components", NULL}; + for (int i = 0; TRANSFORMS[i]; i++) { + if (k8s_helm_gives(doc, TRANSFORMS[i])) { + return NULL; + } + } yyjson_mut_val *charts = yyjson_mut_is_obj(doc) ? yyjson_mut_obj_get(doc, "helmCharts") : NULL; yyjson_mut_val *inline_vals = NULL; size_t idx, max; @@ -4874,9 +4987,12 @@ static void k8s_helm_link_releases(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 if (jd) { yyjson_doc_free(jd); } + h->sensitive = r->sensitive; k8s_helm_release(ctx, b, h, &l, r->src, "terraform", r->chart, rv); + h->sensitive = NULL; cbm_free(CBM_MEM_CLASS_OTHER, r->values); } + cbm_free(CBM_MEM_CLASS_OTHER, r->sensitive); } cbm_free(CBM_MEM_CLASS_OTHER, ctx->tf_releases); ctx->tf_releases = NULL; @@ -4905,7 +5021,7 @@ static void k8s_helm_free(k8s_helm_t *h) { yyjson_mut_doc_free(h->doc); } cbm_free(CBM_MEM_CLASS_OTHER, h->t); - cbm_free(CBM_MEM_CLASS_OTHER, h->dflt.v); + k8s_himgs_free(&h->dflt); cbm_arena_destroy(&h->a); if (h->edges > 0) { cbm_log_info("pass.k8s.helm_images", "edges", itoa_k8s(h->edges)); diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c index 74ab37a2c..a320f6071 100644 --- a/src/pipeline/pass_terraform.c +++ b/src/pipeline/pass_terraform.c @@ -523,15 +523,27 @@ static yyjson_mut_val *tf_json(const tf_t *t, yyjson_mut_doc *doc, TSNode n) { static char *tf_read(const char *path, size_t *len); /* A repository file's text (cbm_alloc; capped and padded like every file - * read), NULL when unreadable -- or when it is no file discovery took in: a - * symlink, or a path through one, is never followed out of the repository. */ + * read; an empty file is empty text), NULL when unreadable -- or when it is + * no file discovery took in: a symlink, or a path through one, is never + * followed out of the repository. */ static char *tf_read_repo(const tf_t *t, const char *path, size_t *len) { if (!cbm_pipeline_file_node(t->ctx->gbuf, t->ctx->project_name, path)) { return NULL; } char full[CBM_SZ_4K]; int n = snprintf(full, sizeof(full), "%s/%s", t->ctx->repo_path, path); - return n > 0 && (size_t)n < sizeof(full) ? tf_read(full, len) : NULL; + if (n <= 0 || (size_t)n >= sizeof(full)) { + return NULL; + } + char *buf = tf_read(full, len); + cbm_path_info_t info; + if (!buf && cbm_path_info_utf8(full, &info) == CBM_PATH_INFO_OK && info.is_regular && + !info.is_symlink && info.size == 0) { + enum { TF_EMPTY_PAD = 16 }; + buf = cbm_calloc(CBM_MEM_CLASS_OTHER, TF_EMPTY_PAD); + *len = 0; + } + return buf; } /* A `values = [...]` item as values: a YAML string / heredoc, file() @@ -633,8 +645,8 @@ static yyjson_mut_val *tf_set_block(const tf_t *t, yyjson_mut_doc *doc, TSNode b return entry; } -/* Is block `a` a `dynamic "set"` / `dynamic "set_list"` -- entries made by a - * for_each, never literal? */ +/* Is block `a` a `dynamic "set"` / `"set_list"` / `"set_sensitive"` -- + * entries (and names) made by a for_each, never literal? */ static bool tf_dynamic_set(const tf_t *t, TSNode a) { TSNode kw = ts_node_named_child(a, 0); TSNode label = ts_node_named_child(a, SKIP_ONE); @@ -649,17 +661,72 @@ static bool tf_dynamic_set(const tf_t *t, TSNode a) { uint32_t lb = ts_node_start_byte(label); size_t ll = ts_node_end_byte(label) - lb; return (ll == strlen("\"set\"") && strncmp(t->src + lb, "\"set\"", ll) == 0) || - (ll == strlen("\"set_list\"") && strncmp(t->src + lb, "\"set_list\"", ll) == 0); + (ll == strlen("\"set_list\"") && strncmp(t->src + lb, "\"set_list\"", ll) == 0) || + (ll == strlen("\"set_sensitive\"") && + strncmp(t->src + lb, "\"set_sensitive\"", ll) == 0); +} + +/* Is node n an identifier spelling `word`? */ +static bool tf_word(const tf_t *t, TSNode n, const char *word) { + if (ts_node_is_null(n)) { + return false; + } + uint32_t b = ts_node_start_byte(n); + size_t l = ts_node_end_byte(n) - b; + return l == strlen(word) && strncmp(t->src + b, word, l) == 0; +} + +/* The expression of attribute `name` of a block's body or of an HCL object + * (`{ name = ..., value = ... }`); a null node when there is none. */ +static TSNode tf_entry_attr(const tf_t *t, TSNode entry, const char *name) { + TSNode e = tf_unwrap(entry); + for (uint32_t j = 0; j < ts_node_named_child_count(e); j++) { + TSNode c = ts_node_named_child(e, j); + if (tf_type(e, "block") && tf_type(c, "body")) { + for (uint32_t k = 0; k < ts_node_named_child_count(c); k++) { + TSNode at = ts_node_named_child(c, k); + if (tf_type(at, "attribute") && ts_node_named_child_count(at) >= PAIR_LEN && + tf_word(t, ts_node_named_child(at, 0), name)) { + return ts_node_named_child(at, SKIP_ONE); + } + } + } else if (tf_type(e, "object") && tf_type(c, "object_elem")) { + TSNode kn = tf_unwrap(ts_node_child_by_field_name(c, "key", (uint32_t)strlen("key"))); + TSNode id = tf_type(kn, "variable_expr") ? ts_node_named_child(kn, 0) : (TSNode){0}; + if (tf_word(t, id, name)) { + return ts_node_child_by_field_name(c, "val", (uint32_t)strlen("val")); + } + } + } + return (TSNode){0}; +} + +/* The literal name of set_sensitive entry `entry` (a block, or an object of + * the provider v3 list) appended to names -- the name only, the value is + * never read; false when the name is no literal. */ +static bool tf_sensitive_name(const tf_t *t, yyjson_mut_doc *doc, yyjson_mut_val *names, + TSNode entry) { + yyjson_mut_val *nm = tf_json(t, doc, tf_entry_attr(t, entry, "name")); + if (!yyjson_mut_is_str(nm)) { + return false; + } + yyjson_mut_arr_append(names, nm); + return true; } /* The values a helm_release body gives its chart -- `values` items * coalesced in order, then each `set` -- as JSON (cbm_alloc); NULL when a - * part is no literal, or is made by a `dynamic` block. set_sensitive is - * never read. */ -static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles) { + * part is no literal, is made by a `dynamic` block, or a postrender may + * rewrite what it deploys. set_sensitive values are never read: their names + * go to *sensitive (a JSON array, cbm_alloc; NULL: none), and an image that + * reads one is not known (pass_k8s.c). */ +static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles, + char **sensitive) { + *sensitive = NULL; yyjson_mut_doc *doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; - bool ok = root != NULL; + yyjson_mut_val *names = doc ? yyjson_mut_arr(doc) : NULL; + bool ok = root != NULL && names != NULL; for (int pass = 0; ok && pass < PAIR_LEN; pass++) { for (uint32_t i = 0; ok && i < ts_node_named_child_count(body); i++) { TSNode a = ts_node_named_child(body, i); @@ -669,12 +736,30 @@ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, bool is_set = kl == strlen("set") && strncmp(t->src + kb, "set", kl) == 0; bool is_values = kl == strlen("values") && strncmp(t->src + kb, "values", kl) == 0; /* set_list and dynamic set values are not read: the release's - * values would be partial */ + * values would be partial; a postrender may rewrite its images */ if ((kl == strlen("set_list") && strncmp(t->src + kb, "set_list", kl) == 0) || - tf_dynamic_set(t, a)) { + tf_dynamic_set(t, a) || (tf_type(a, "block") && tf_word(t, key, "postrender"))) { ok = false; break; } + bool is_sensitive = + kl == strlen("set_sensitive") && strncmp(t->src + kb, "set_sensitive", kl) == 0; + if (pass == 0 && is_sensitive && tf_type(a, "block")) { + ok = tf_sensitive_name(t, doc, names, a); + continue; + } + if (pass == 0 && is_sensitive && tf_type(a, "attribute")) { + /* provider v3: set_sensitive = [{ name, value }, ...] */ + TSNode list = tf_unwrap(ts_node_named_child(a, SKIP_ONE)); + ok = tf_type(list, "tuple"); + for (uint32_t j = 0; ok && j < ts_node_named_child_count(list); j++) { + TSNode it = ts_node_named_child(list, j); + if (!tf_type(it, "tuple_start") && !tf_type(it, "tuple_end")) { + ok = tf_sensitive_name(t, doc, names, it); + } + } + continue; + } if (pass == 0 && is_values && tf_type(a, "attribute")) { TSNode list = tf_unwrap(ts_node_named_child(a, SKIP_ONE)); ok = tf_type(list, "tuple"); @@ -706,6 +791,15 @@ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, if (ok) { size_t len = 0; out = yyjson_mut_val_write_opts(root, 0, cbm_json_writer_allocator(), &len, NULL); + if (out && yyjson_mut_arr_size(names) > 0) { + *sensitive = + yyjson_mut_val_write_opts(names, 0, cbm_json_writer_allocator(), &len, NULL); + if (!*sensitive) { + /* the names not kept: no image of the release could be told apart */ + cbm_free(CBM_MEM_CLASS_OTHER, out); + out = NULL; + } + } } if (doc) { yyjson_mut_doc_free(doc); @@ -727,8 +821,9 @@ static void tf_release_push(tf_t *t, int64_t src, int64_t chart, TSNode body) { } const char **files = NULL; int nfiles = 0; - ctx->tf_releases[ctx->tf_release_n++] = - (cbm_tf_release_t){src, chart, tf_release_values(t, body, &files, &nfiles)}; + char *sensitive = NULL; + char *values = tf_release_values(t, body, &files, &nfiles, &sensitive); + ctx->tf_releases[ctx->tf_release_n++] = (cbm_tf_release_t){src, chart, values, sensitive}; /* the values files it reads, like a kustomization's valuesFile: an edit of * one alone renders the release again (pipeline_incremental.c) */ for (int i = 0; i < nfiles; i++) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 553aa19d9..402474c20 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -385,9 +385,10 @@ typedef struct { * when a part is not a literal (an interpolation, a variable). pass_k8s.c * renders the chart's images with them. */ typedef struct cbm_tf_release { - int64_t src; /* the resource.helm_release node */ - int64_t chart; /* the Chart node */ - char *values; /* owned (cbm_free); NULL: unknown */ + int64_t src; /* the resource.helm_release node */ + int64_t chart; /* the Chart node */ + char *values; /* owned (cbm_free); NULL: unknown */ + char *sensitive; /* owned: JSON array of its set_sensitive names (never values); NULL: none */ } cbm_tf_release_t; /* Shared context passed to each pass function. diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 2227ac308..7b89c755a 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -4871,6 +4871,78 @@ TEST(probe_helm_release_terraform_dynamic_set) { PASS(); } +/* A set_sensitive whose value is never read: a release image reading its + * name (image.tag) is not known; one reading none of them still is. A + * postrender may rewrite what a release deploys: no image of its own. */ +TEST(probe_helm_release_terraform_sensitive_and_postrender) { + static const GpfFile files[] = { + GPF_APP_CHART, + {"tf/main.tf", + "resource \"helm_release\" \"tagged\" {\n name = \"tagged\"\n" + " chart = \"../charts/app\"\n set {\n name = \"image.repository\"\n" + " value = \"quay.io/acme/app\"\n }\n set_sensitive {\n name = \"image.tag\"\n" + " value = var.tag\n }\n}\n" + "resource \"helm_release\" \"pw\" {\n name = \"pw\"\n" + " chart = \"../charts/app\"\n set {\n name = \"image.repository\"\n" + " value = \"quay.io/acme/pw\"\n }\n set_sensitive {\n name = \"db.password\"\n" + " value = var.pw\n }\n}\n" + "resource \"helm_release\" \"post\" {\n name = \"post\"\n" + " chart = \"../charts/app\"\n set {\n name = \"image.tag\"\n" + " value = \"3.3\"\n }\n postrender {\n binary_path = \"./kustomize.sh\"\n }\n}\n"}}; + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.repository\",\"image.tag\"]\n" + "Class:resource.helm_release.pw -> Image:quay.io/acme/pw DEPLOYS " + "image=quay.io/acme/pw:1.0 via=terraform ref=image override_of=[\"image.repository\"]\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " + "ref=image default_of=[\"image.repository\",\"image.tag\"]"; + if (!gpf_expect_image_edges(files, 4, want)) { + FAIL("set_sensitive names and postrender make release images unknown"); + } + PASS(); +} + +/* A kustomization that also transforms what it inflates (images:, patches) + * has no release image of its own: the transformer may change it. */ +TEST(probe_helm_release_kustomize_transformers) { + static const GpfFile files[] = { + {"kz/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nhelmCharts:\n" + " - name: app\n releaseName: kapp\n valuesInline:\n image:\n" + " tag: \"1.5\"\nimages:\n - name: ghcr.io/acme/app\n newTag: \"2.0\"\n"}, + {"kz/charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\n"}, + {"kz/charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"kz/charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"}}; + if (!gpf_expect_image_edges(files, 4, GPF_APP_DEFAULTS)) { + FAIL("a kustomization's own transformers make its release images unknown"); + } + PASS(); +} + +/* An empty values.yaml is values {} (Helm's), not values not read: the + * subchart under it still renders its image. */ +TEST(probe_helm_empty_values_file_renders) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", + "apiVersion: v2\nname: app\nversion: 1.0.0\nappVersion: \"1.0\"\n"}, + {"charts/app/values.yaml", ""}, + {"charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " spec:\n containers:\n - name: app\n" + " image: \"ghcr.io/acme/{{ .Chart.Name }}:{{ .Chart.AppVersion }}\"\n"}}; + static const char want[] = + "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image\n" + "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " + "ref=image"; + if (!gpf_expect_image_edges(files, 3, want)) { + FAIL("an empty values.yaml renders with {}"); + } + PASS(); +} + /* A subchart a dependency aliases renders with .Chart.Name its alias (Helm's * getAliasDependency), so an image naming the chart names the alias. */ TEST(probe_helm_aliased_subchart_chart_name) { @@ -4919,8 +4991,10 @@ TEST(probe_helm_list_path_mark_on_chart_edge) { PASS(); } +#ifndef _WIN32 /* gpf_image_edges with symlinks laid in after the files: links[i] names a - * repository path and the absolute or relative target it points at. */ + * repository path and the absolute or relative target it points at (POSIX + * symlinks: the tests using it skip on Windows). */ static char *gpf_image_edges_linked(const GpfFile *files, int nfiles, const GpfFile *links, int nlinks) { GpfProj lp; @@ -4975,6 +5049,7 @@ static char *gpf_image_edges_linked(const GpfFile *files, int nfiles, const GpfF gpf_cleanup(&lp, store); return out; } +#endif /* A values file a chart, a kustomization or a Terraform file() names by path * is read only when it is a file of the repository: a symlink out of it -- @@ -4982,6 +5057,9 @@ static char *gpf_image_edges_linked(const GpfFile *files, int nfiles, const GpfF * file is such a link renders nothing (its values are unknown, not empty); * the releases reading one have no image of their own. */ TEST(probe_helm_values_symlink_not_followed) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX symlink contract"); +#else char outside[256]; snprintf(outside, sizeof(outside), "/tmp/cbm_gpf_out_XXXXXX"); ASSERT_NOT_NULL(cbm_mkdtemp(outside)); @@ -5024,11 +5102,15 @@ TEST(probe_helm_values_symlink_not_followed) { FAIL("no values read through a symlink out of the repository"); } PASS(); +#endif } /* A Terraform helm_release reading its values with file() of a symlink out * of the repository has no image of its own: the link is never followed. */ TEST(probe_helm_terraform_values_symlink_not_followed) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX symlink contract"); +#else char outside[256]; snprintf(outside, sizeof(outside), "/tmp/cbm_gpf_out_XXXXXX"); ASSERT_NOT_NULL(cbm_mkdtemp(outside)); @@ -5057,6 +5139,7 @@ TEST(probe_helm_terraform_values_symlink_not_followed) { FAIL("no Terraform values read through a symlink out of the repository"); } PASS(); +#endif } /* A build argument holding a secret -- by its name (*_TOKEN) or its value (a @@ -6929,6 +7012,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_helm_release_flux_unread_sources); RUN_TEST(probe_helm_release_kustomize_unread_sources); RUN_TEST(probe_helm_release_terraform_dynamic_set); + RUN_TEST(probe_helm_release_terraform_sensitive_and_postrender); + RUN_TEST(probe_helm_release_kustomize_transformers); + RUN_TEST(probe_helm_empty_values_file_renders); RUN_TEST(probe_helm_aliased_subchart_chart_name); RUN_TEST(probe_helm_list_path_mark_on_chart_edge); RUN_TEST(probe_helm_values_symlink_not_followed); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index e86fd2e77..dc03c8cf7 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4559,6 +4559,191 @@ TEST(helm_change_of_a_new_value_spoils_nothing) { PASS(); } +/* An image line inside a range: n captures of one probe cost memory linear + * in n (the capture arrays grow by doubling), not a copy of every earlier + * capture each time. */ +TEST(helm_render_captures_grow_linearly) { + enum { ITER = 5000 }; + static const char tpl[] = "{{- range 5000 }}\n" + "image: \"ghcr.io/acme/app:1\"\n" + "{{ end }}\n"; + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_4K); + cbm_helm_probe_t probe = {.line = 2}; + size_t before = cbm_arena_total(&a); + cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), NULL, NULL, NULL, &probe, 1); + size_t used = cbm_arena_total(&a) - before; + int n = probe.n; + cbm_arena_destroy(&a); + ASSERT_EQ(n, ITER); + ASSERT_TRUE(used < (size_t)CBM_SZ_16 * CBM_SZ_1K * CBM_SZ_1K); /* far below 8*n^2 bytes */ + PASS(); +} + +/* Memory a render takes -- here a 20,000-item values list copied by each of + * 200 `first` calls -- counts against its budget: past 256 MiB it stops and + * the template's captures are UNKNOWN. */ +TEST(helm_render_memory_budget_ends_unknown) { + enum { ITEMS = 20000 }; + size_t cap = (size_t)ITEMS * 4 + CBM_SZ_64; + char *vals = malloc(cap); + ASSERT_NOT_NULL(vals); + size_t o = (size_t)snprintf(vals, cap, "tag: \"1.0\"\nbig: ["); + for (int i = 0; i < ITEMS; i++) { + o += (size_t)snprintf(vals + o, cap - o, "%s1", i ? "," : ""); + } + snprintf(vals + o, cap - o, "]\n"); + static const char tpl[] = "image: \"ghcr.io/acme/app:{{ .Values.tag }}\"\n" + "{{- range 200 }}{{ $x := first $.Values.big }}{{ end }}\n"; + char *got = helm_capture(vals, tpl, NULL, 1); + free(vals); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A set skipped by an UNKNOWN condition whose target is a call's result -- + * `index .Values "image"`, the idiom for a dashed key -- may change the map + * it names: what the image reads of it is UNKNOWN. */ +TEST(helm_skipped_set_of_a_call_result) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + static const char tpl[] = + "{{- if semverCompare \">=1.29-0\" .Capabilities.KubeVersion.Version }}" + "{{- $_ := set (index .Values \"image\") \"tag\" \"2.0\" }}{{- end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A new map given a map another part holds is new no more: a change made + * through it (`set $img ...` of `index $d "image"`) reaches .Values. */ +TEST(helm_own_map_given_a_shared_one) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + static const char tpl[] = "{{- $d := dict -}}{{- $_ := set $d \"image\" .Values.image -}}\n" + "{{- $img := index $d \"image\" -}}{{- $_ := set $img \"tag\" \"2.0\" -}}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A set whose target is an UNKNOWN call result (index by .Release.Name) may + * change any map of the values: every image after it is UNKNOWN. */ +TEST(helm_set_of_an_unknown_call_result) { + static const char vals[] = + "apps: {web: {repository: ghcr.io/acme/web, tag: \"1.0\"}}\n"; + static const char tpl[] = "{{- $_ := set (index .Values.apps .Release.Name) \"tag\" \"2.0\" }}\n" + "image: \"{{ .Values.apps.web.repository }}:{{ .Values.apps.web.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A break under an UNKNOWN condition: whether the rest of the loop ran is + * unknown, so what it assigns ($tag = .) is UNKNOWN after the loop. */ +TEST(helm_unknown_break_spoils_the_rest_of_the_loop) { + static const char vals[] = "versions: [\"1.0\", \"2.0\"]\n"; + static const char tpl[] = + "{{- $tag := \"0.9\" }}{{- range .Values.versions }}\n" + "{{- if semverCompare \">=1.29-0\" $.Capabilities.KubeVersion.Version }}{{ break }}" + "{{ end }}\n" + "{{- $tag = . }}{{- end }}\n" + "image: \"ghcr.io/acme/app:{{ $tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 4); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A variable once made from `dict` and then (in a body an UNKNOWN condition + * skips) redefined or given .Values.image is no new map: a set of it there + * may change .Values. Without the redefinition the set is of the new map. */ +TEST(helm_skipped_set_after_redefinition) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + static const char redefined[] = + "{{- $cfg := dict }}{{- if .Capabilities.APIVersions }}" + "{{- $cfg := .Values.image }}{{- $_ := set $cfg \"tag\" \"2.0\" }}{{- end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + static const char assigned[] = + "{{- $cfg := dict }}{{- if .Capabilities.APIVersions }}" + "{{- $cfg = .Values.image }}{{- $_ := set $cfg \"tag\" \"2.0\" }}{{- end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + static const char kept[] = "{{- $cfg := dict }}{{- if .Capabilities.APIVersions }}" + "{{- $_ := set $cfg \"tag\" \"2.0\" }}{{- end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, redefined, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + got = helm_capture(vals, assigned, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + got = helm_capture(vals, kept, NULL, 2); + ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + free(got); + PASS(); +} + +/* Many maps poisoned (a new dict changed each iteration) keep every later + * read correct: a values read after them is known. */ +TEST(helm_many_changed_maps_keep_reads_right) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + static const char tpl[] = "{{- range 20000 }}{{ $c := dict }}{{ $_ := set $c \"k\" \"v\" }}" + "{{ end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 2); + ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + free(got); + PASS(); +} + +/* Defines that include each other: A changes its argument and includes B, + * B includes A. Whether B may change its argument is not settled by A's + * scan still running -- an UNKNOWN include of B with .Values.image spoils + * what reads it. */ +TEST(helm_define_memo_under_mutual_recursion) { + static const char defs_src[] = + "{{- define \"A\" -}}{{- include \"B\" . -}}{{- $_ := set . \"tag\" \"2.0\" -}}{{- end -}}\n" + "{{- define \"B\" -}}{{- if false }}{{ include \"A\" . }}{{ end -}}{{- end -}}\n"; + cbm_helm_defines_t *d = cbm_helm_defines_new(); + ASSERT_NOT_NULL(d); + cbm_helm_defines_add(d, "templates/_ab.tpl", defs_src, strlen(defs_src)); + static const char vals[] = + "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\nother: {tag: \"x\"}\n"; + static const char tpl[] = + "{{- if .Capabilities.APIVersions }}{{ include \"A\" .Values.other }}{{ end }}\n" + "{{- if .Capabilities.APIVersions }}{{ include \"B\" .Values.image }}{{ end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, d, 3); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* An include not run because its argument is poisoned may still change maps + * within it: a handle taken before ($sub) reads UNKNOWN after. */ +TEST(helm_include_with_poisoned_argument) { + static const char defs_src[] = + "{{- define \"mut\" -}}{{- $_ := set .sub \"tag\" \"2.0\" -}}{{- end -}}\n"; + cbm_helm_defines_t *d = cbm_helm_defines_new(); + ASSERT_NOT_NULL(d); + cbm_helm_defines_add(d, "templates/_mut.tpl", defs_src, strlen(defs_src)); + static const char vals[] = "image: {sub: {repository: ghcr.io/acme/app, tag: \"1.0\"}}\n"; + static const char tpl[] = "{{- $sub := .Values.image.sub -}}\n" + "{{- $_ := set .Values.image \"x\" \"y\" -}}\n" + "{{- include \"mut\" .Values.image -}}\n" + "image: \"{{ $sub.repository }}:{{ $sub.tag }}\"\n"; + char *got = helm_capture(vals, tpl, d, 4); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + /* A compose project's .env feeds its compose file (variables in image, * build): changing it is no closure repair -- the unchanged compose file's * services would keep the old image. It rebuilds; no stale image remains. */ @@ -24297,6 +24482,16 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(helm_mutator_spoils_the_map_it_changes); RUN_TEST(helm_unknown_break_leaves_the_loop); RUN_TEST(helm_change_of_a_new_value_spoils_nothing); + RUN_TEST(helm_render_captures_grow_linearly); + RUN_TEST(helm_render_memory_budget_ends_unknown); + RUN_TEST(helm_skipped_set_of_a_call_result); + RUN_TEST(helm_own_map_given_a_shared_one); + RUN_TEST(helm_set_of_an_unknown_call_result); + RUN_TEST(helm_unknown_break_spoils_the_rest_of_the_loop); + RUN_TEST(helm_skipped_set_after_redefinition); + RUN_TEST(helm_many_changed_maps_keep_reads_right); + RUN_TEST(helm_define_memo_under_mutual_recursion); + RUN_TEST(helm_include_with_poisoned_argument); RUN_TEST(incremental_compose_dotenv_change_rebuilds); RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges); RUN_TEST(incremental_second_build_declarer_keeps_the_edge); From aa4775f5669092fde88142c2011e82649fd8466c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 01:38:07 +0200 Subject: [PATCH 12/19] test(doc_links_pdf): receiver-qualified Go method nodes (#1913) Two PDF doc-link tests named a Go method by its flat QN. With #1913 a Go method's QN carries its receiver, so the lookups found no node. The resolver is unchanged: for a Go method, rel(parent_class).name now equals the QN, so the method binds through its QN, like a member of any other language. - pdf_links_pipeline: page 1 names `server.Handler.Serve`; the edge target pkg.server.Serve becomes pkg.server.Handler.Serve. Same assertions: pdf_qn syntax, unique tier. - pdf_qn_method_through_owner, page 2: `time.Provider.Now` binds pkg.time.Provider.Now (was pkg.time.Now), syntax pdf_qn. - pdf_qn_method_through_owner, page 1: `time.Now` must not bind the method. The target moves to pkg.time.Provider.Now too. Against the old flat QN, no node exists any more, so the zero count would have passed vacuously. The page 2 assertion proves that the node exists. The ownerless-QN rule in doc_links_pdf.c stays: a templ method keeps the flat package QN while it carries parent_class (extract_defs.c only receiver-qualifies CBM_LANG_GO). Its comment no longer says that Go methods have no receiver. Signed-off-by: Martin Vogel --- src/pipeline/doc_links_pdf.c | 7 ++++--- tests/test_doc_links_pdf.c | 15 +++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/pipeline/doc_links_pdf.c b/src/pipeline/doc_links_pdf.c index a667fb6c7..1eaf057ae 100644 --- a/src/pipeline/doc_links_pdf.c +++ b/src/pipeline/doc_links_pdf.c @@ -328,9 +328,10 @@ static void qn_candidate(const pdfr_index_t *x, const cbm_gbuf_node_t *n, const const char *rel = rel_qn(x, n->qualified_name); char alt[PDFR_KEY_CAP]; bool owned = owner_name(x, n, alt, sizeof(alt)); - /* A member whose QN leaves its owner out (a Go method's has no receiver) - * is named through the owner: `time.Now` is package time's function, - * never the method Now of a type that package time declares. */ + /* A member whose QN leaves its owner out (a templ method's keeps the flat + * package QN; a Go method's did before index format 4) is named through + * the owner: `time.Now` is package time's function, never the method Now + * of a type that package time declares. */ bool ownerless_qn = owned && !comp_suffix(rel, alt); bool via_rel = !ownerless_qn && comp_suffix(rel, s); bool via_alt = !via_rel && owned && strcmp(alt, rel) != 0 && comp_suffix(alt, s); diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c index 1a78b7254..9c7bf90ff 100644 --- a/tests/test_doc_links_pdf.c +++ b/tests/test_doc_links_pdf.c @@ -1087,8 +1087,9 @@ TEST(pdf_links_pipeline) { ASSERT_NOT_NULL(strstr(props, "\"line\":1")); /* qualified names (pdf_qn) passed a second held-out audit (97 of 99 * correct, Wilson 95 % low 0.929) once a member whose QN leaves its owner - * out binds only through the owner. A Go method through its receiver: */ - dm_edge(db, "docs.design.page_1", "pkg.server.Serve", props, sizeof(props), &n); + * out binds only through the owner. A Go method through its receiver + * (receiver-qualified QN since index format 4): */ + dm_edge(db, "docs.design.page_1", "pkg.server.Handler.Serve", props, sizeof(props), &n); ASSERT_EQ(n, 1); ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_qn\"")); ASSERT_NOT_NULL(strstr(props, "\"tier\":\"unique\"")); @@ -1142,10 +1143,12 @@ TEST(pdf_links_pipeline) { PASS(); } -/* A held-out finding of the PDF audit: a Go method's QN has no receiver, so +/* A held-out finding of the PDF audit: a Go method's QN had no receiver, so * `time.Now` (the standard library's function, in a spec) matched the method * Now of a type that the repository's own package time declares. A method is - * named through its owner: `time.Provider.Now` still binds it. */ + * named through its owner: `time.Provider.Now` still binds it. Since index + * format 4 the QN carries the receiver (pkg.time.Provider.Now), and both + * directions are checked against that node. */ TEST(pdf_qn_method_through_owner) { cbm_doclink_test_set_ships(CBM_DOCLINK_PDF_QN, true); char tmp[256]; @@ -1169,9 +1172,9 @@ TEST(pdf_qn_method_through_owner) { ASSERT_EQ(dm_index(repo, db, NULL), 0); char props[512]; int n; - dm_edge(db, "docs.spec.page_1", "pkg.time.Now", props, sizeof(props), &n); + dm_edge(db, "docs.spec.page_1", "pkg.time.Provider.Now", props, sizeof(props), &n); ASSERT_EQ(n, 0); - dm_edge(db, "docs.spec.page_2", "pkg.time.Now", props, sizeof(props), &n); + dm_edge(db, "docs.spec.page_2", "pkg.time.Provider.Now", props, sizeof(props), &n); ASSERT_EQ(n, 1); ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_qn\"")); th_cleanup(tmp); From 3401def5ba3acecdfb636e12a25f6051bb44ed4e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 01:59:48 +0200 Subject: [PATCH 13/19] fix(deploy): coarse rule for Helm changes not run, BasePath includes, render memory Review round 3 of #2677. The user decided on a rule sound by construction: a change of maps the render does not run exactly leaves every later image of that render UNKNOWN, with no exemption. Changes of maps (helm_template.c): - Under UNKNOWN control -- a skipped if / with / range body, the rest of a loop an UNKNOWN break / continue may leave, an include or template not run, a tpl not run -- any set / unset / merge (the must* forms too), any tpl, and any include / template of a name not known, of no define or template file of the chart, or of one that may change maps, leaves the images after it UNKNOWN. - An executed set / merge does too unless its target is a known map that is poisoned exactly; a merge also for a target already poisoned, the .Values root, or a map not known within it. The own provenance and the freshness scan are gone. - .Template.BasePath is modelled as Helm sets it: /templates, the top chart's name, then /charts/ per subchart (pass_k8s.c). An include naming one of the chart's template files by it (the checksum idiom) is scanned with the same rule; a file reached again while scanned, or no such file, may change maps. Bounds (helm_template.c): - A capture's keys are a hash set and its mark is built in one pass (one sort for the prefix check), its steps charged. - Each render works in an arena of its own; the caller keeps its results only. - The trip reason is recorded where a budget trips: an output trip is logged as output. Releases (pass_terraform.c, pass_k8s.c): - Any set_sensitive leaves the release without an image of its own (the name-overlap check is gone), and so does a postrender written as a dynamic block or an attribute. Cost on the 20 infra corpora against 5c5964ef: 20 Image edges lost, 0 added, no budget trip. 5 are changes the rule rightly cannot follow (a second mustMerge into a poisoned values map; a skipped part that may change maps), 15 come from checksum-included files that hold a tpl. Two controls of rounds 1-3 now read UNKNOWN; their tests say so. Tests: one per item, each failing on 5c5964ef by its assertion (the BasePath ones also on the coarse rule without it); the wrong-image scenarios of rounds 2 and 3 stay as tests. Signed-off-by: Martin Vogel --- src/pipeline/helm_template.c | 1009 +++++++++++------------------- src/pipeline/helm_template.h | 17 +- src/pipeline/pass_k8s.c | 114 +--- src/pipeline/pass_terraform.c | 100 +-- src/pipeline/pipeline_internal.h | 7 +- tests/test_grammar_probe_f.c | 110 +++- tests/test_pipeline.c | 341 +++++++++- 7 files changed, 843 insertions(+), 855 deletions(-) diff --git a/src/pipeline/helm_template.c b/src/pipeline/helm_template.c index 2f96db45d..5004c2e7d 100644 --- a/src/pipeline/helm_template.c +++ b/src/pipeline/helm_template.c @@ -1049,6 +1049,16 @@ void cbm_helm_defines_free(cbm_helm_defines_t *d) { cbm_free(CBM_MEM_CLASS_OTHER, d); } +/* Template file `rel` of the set; NULL: none. */ +static const hd_file_t *hd_file_find(const cbm_helm_defines_t *d, const char *rel) { + for (int i = 0; d && i < d->nfiles; i++) { + if (strcmp(d->files[i]->rel, rel) == 0) { + return d->files[i]; + } + } + return NULL; +} + static const hd_def_t *hd_find(const cbm_helm_defines_t *d, const char *name) { return d && d->names ? cbm_ht_get(d->names, name) : NULL; } @@ -1250,12 +1260,9 @@ struct hv { hv_map_t *m; hv_list_t *l; const char *path; /* the values path it holds; NULL: none */ - bool own; /* a new value holding nothing another part holds (fromYaml, deepCopy, - * a part of one; known or not): changing it changes nothing else */ }; static const hv_t HV_UNKNOWN = {.k = HV_UNK}; -static const hv_t HV_UNKNOWN_OWN = {.k = HV_UNK, .own = true}; static hv_t hv_nil(void) { return (hv_t){.k = HV_NIL}; @@ -1377,17 +1384,23 @@ typedef struct { bool brk; bool cont; long steps; - size_t written; /* bytes appended to any sink */ - size_t arena0; /* the arena's total when the render began */ - bool stop; /* over HELM_MAX_STEPS / _OUTPUT / _MEMORY: the render is UNKNOWN */ + size_t written; /* bytes appended to any sink */ + size_t arena0; /* the arena's total when the render began */ + bool stop; /* over HELM_MAX_STEPS / _OUTPUT / _MEMORY: the render is UNKNOWN */ + const char *why; /* which budget stopped it */ + const char *rel; /* the template's path (logs) */ /* maps a set / unset / merge changed in place (not modelled): what they * hold is UNKNOWN from then on -- by address ("%p" keys in the arena), * and for a merge of values also by values path (everything below it) */ CBMHashTable *poison_maps; CBMHashTable *poison_paths; - bool tainted; /* a change of unknown target: every capture ending later is UNKNOWN */ - CBMHashTable *mutates; /* define name -> it may change a map (he_define_may_mutate) */ - long busy_hits; /* defines met again while being scanned */ + /* a change of maps not run exactly: every capture ending later is UNKNOWN */ + bool tainted; + CBMHashTable *mutates; /* define name -> it may change a map (he_define_may_mutate) */ + CBMHashTable *file_mutates; /* template file -> the same (he_file_may_mutate) */ + const char *base_path; /* .Template.BasePath; NULL: not known */ + const char *chart_dir; /* where the chart's template files are (meta.dir) */ + long busy_hits; /* defines met again while being scanned */ /* probes of the top-level template */ const char *top_src; size_t top_len; @@ -1400,6 +1413,7 @@ typedef struct { const char **ckeys; int nck; int ckcap; + CBMHashTable *ckset; /* ckeys as a set */ TSParser *parser; TSTree **trees; int ntrees; @@ -1411,24 +1425,47 @@ static bool hst_over_memory(const hst_t *st) { return cbm_arena_total(st->a) - st->arena0 > HELM_MAX_MEMORY; } +/* The render stops: budget `why` is spent. */ +static void hst_trip(hst_t *st, const char *why) { + if (!st->stop) { + st->stop = true; + st->why = why; + } +} + /* One step of work; false once the render is over its budget. */ static bool hst_step(hst_t *st) { - if (!st->stop && (++st->steps > HELM_MAX_STEPS || hst_over_memory(st))) { - st->stop = true; + if (!st->stop && ++st->steps > HELM_MAX_STEPS) { + hst_trip(st, "steps"); + } else if (!st->stop && hst_over_memory(st)) { + hst_trip(st, "memory"); } return !st->stop; } /* n more bytes of output; false (and the render stopped) past the budget. */ static bool hst_write(hst_t *st, size_t n) { - if (st->stop || n > HELM_MAX_OUTPUT - st->written || hst_over_memory(st)) { - st->stop = true; + if (!st->stop && n > HELM_MAX_OUTPUT - st->written) { + hst_trip(st, "output"); + } else if (!st->stop && hst_over_memory(st)) { + hst_trip(st, "memory"); + } + if (st->stop) { return false; } st->written += n; return true; } +/* A change of maps the render does not run exactly (see he_call): from here + * on no capture is known. */ +static void hst_taint(hst_t *st, const char *cause) { + if (!st->tainted) { + st->tainted = true; + cbm_log_debug("helm.render.taint", "file", st->rel ? st->rel : "", "cause", cause); + } +} + static void hs_append(hst_t *st, hsink_t *s, const char *p, size_t n) { if (s->unk || n == 0) { return; @@ -1466,15 +1503,21 @@ static void hst_record(hst_t *st, const char *path) { if (st->capture < 0 || !path) { return; } - for (int i = 0; i < st->nck; i++) { - if (strcmp(st->ckeys[i], path) == 0) { - return; - } + if (!st->ckset) { + st->ckset = cbm_ht_create(0); + } + if (!st->ckset) { + st->cap.unk = true; /* a key read not kept: the capture's mark is not whole */ + return; + } + if (cbm_ht_get(st->ckset, path)) { + return; } if (st->nck == st->ckcap) { int cap = st->ckcap ? st->ckcap * PAIR_LEN : CBM_SZ_16; const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); if (!g) { + st->cap.unk = true; return; } if (st->nck) { @@ -1483,6 +1526,7 @@ static void hst_record(hst_t *st, const char *path) { st->ckeys = g; st->ckcap = cap; } + cbm_ht_set(st->ckset, path, (void *)(uintptr_t)SKIP_ONE); st->ckeys[st->nck++] = path; } @@ -1603,9 +1647,10 @@ static bool hst_set_add(hst_t *st, CBMHashTable **set, const char *k) { static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth); -/* Map v changed in place; false when v is no map (or no room to say so). A - * merge (`deep`) may change every map within it too -- a map the template - * made (dict) holds others by reference, so those are poisoned as well. */ +/* Map v changed in place; false when v is no map, when (`deep`, a merge) + * a map within it cannot be poisoned whole, or with no room to say so. A + * merge may change every map within it too -- a map the template made (dict) + * holds others by reference, so those are poisoned as well. */ static bool hst_poison(hst_t *st, const hv_t *v, bool deep) { return hst_poison_at(st, v, deep, 0); } @@ -1614,27 +1659,22 @@ static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { if (depth > HELM_MAX_NESTING || !hst_step(st)) { return false; } - if (deep && v->k == HV_LIST) { - /* a list itself is never changed in place; the maps in it may be */ - for (int i = 0; i < v->l->n; i++) { - if ((hv_is_map(&v->l->items[i]) || hv_is_list(&v->l->items[i])) && - !hst_poison_at(st, &v->l->items[i], deep, depth + SKIP_ONE)) { - return false; - } - } - return true; - } - /* a values list (deep): by its path, which every item's path extends */ - const void *map = v->k == HV_MAP ? (const void *)v->m - : v->k == HV_JSON && (yyjson_mut_is_obj(v->j) || (deep && v->path)) - ? (const void *)v->j - : NULL; + const void *map = v->k == HV_MAP ? (const void *)v->m + : v->k == HV_JSON && yyjson_mut_is_obj(v->j) ? (const void *)v->j + : NULL; if (!map) { return false; } + /* a deep change reaches every map within: one not known, one a change + * already made unknown, the values root (every path below it), or a map + * not of the values (whose maps no path covers) cannot be poisoned whole */ + if (deep && (hv_poisoned(st, v) || (v->path && !v->path[0]) || (v->k == HV_JSON && !v->path))) { + return false; + } if (deep && v->k == HV_MAP) { for (int i = 0; i < v->m->n; i++) { - if (hv_is_map(&v->m->vals[i]) && !hst_poison_at(st, &v->m->vals[i], deep, depth + 1)) { + const hv_t *x = &v->m->vals[i]; + if (x->k == HV_UNK || (hv_is_map(x) && !hst_poison_at(st, x, deep, depth + SKIP_ONE))) { return false; } } @@ -1767,6 +1807,105 @@ static const char *hst_scalar(hst_t *st, const char *s, size_t n) { return n ? cbm_arena_strndup(st->a, s, n) : NULL; } +/* strcmp order of string s against k + "." (lk = strlen(k)): 0 for each s + * that starts with it. */ +static int hst_cmp_below(const char *s, const char *k, size_t lk) { + int r = strncmp(s, k, lk); + if (r != 0) { + return r; + } + unsigned char c = (unsigned char)s[lk]; + return c < '.' ? -SKIP_ONE : c > '.' ? SKIP_ONE : 0; +} + +/* Does sorted (n keys, strcmp order) hold a key below k (k + "." ...)? */ +static bool hst_has_below(const char *const *sorted, int n, const char *k) { + size_t lk = strlen(k); + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (hst_cmp_below(sorted[mid], k, lk) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo < n && hst_cmp_below(sorted[lo], k, lk) == 0; +} + +static int hst_str_cmp(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* The leaves the capture read as a JSON array in the order read (NULL: none, + * or *whole false: not built in full). A path that is the prefix of another + * (at a `.`) is dropped. One sort and one allocation: k keys cost O(k log k) + * steps and their bytes of memory. */ +static const char *hst_capture_keys(hst_t *st, bool *whole) { + *whole = true; + int n = st->nck; + if (n == 0) { + return NULL; + } + const char **sorted = cbm_arena_alloc(st->a, (size_t)n * sizeof(*sorted)); + bool *keep = cbm_arena_alloc(st->a, (size_t)n * sizeof(*keep)); + if (!sorted || !keep) { + *whole = false; + return NULL; + } + memcpy(sorted, st->ckeys, (size_t)n * sizeof(*sorted)); + qsort(sorted, (size_t)n, sizeof(*sorted), hst_str_cmp); + size_t total = PAIR_LEN; /* [ ] */ + int kept = 0; + for (int i = 0; i < n; i++) { + if (!hst_step(st)) { + *whole = false; + return NULL; + } + const char *k = st->ckeys[i]; + keep[i] = k[0] && !hst_has_below(sorted, n, k); + if (!keep[i]) { + continue; + } + total += PAIR_LEN + (kept ? SKIP_ONE : 0); /* "" and , */ + for (const char *c = k; *c; c++) { + total += *c == '"' || *c == '\\' ? PAIR_LEN : SKIP_ONE; + } + kept++; + } + if (kept == 0) { + return NULL; + } + char *json = cbm_arena_alloc(st->a, total + SKIP_ONE); + if (!json) { + *whole = false; + return NULL; + } + size_t o = 0; + json[o++] = '['; + for (int i = 0; i < n; i++) { + if (!keep[i]) { + continue; + } + if (o > SKIP_ONE) { + json[o++] = ','; + } + json[o++] = '"'; + for (const char *c = st->ckeys[i]; *c; c++) { + if (*c == '"' || *c == '\\') { + json[o++] = '\\'; + } + json[o++] = *c; + } + json[o++] = '"'; + } + json[o++] = ']'; + json[o] = '\0'; + return json; +} + +/* The capture (if any) ends: its image and the keys it read join its probe. */ static void hst_capture_end(hst_t *st) { if (st->capture < 0) { return; @@ -1787,50 +1926,21 @@ static void hst_capture_end(hst_t *st) { p->cap = cap; } } - const char **img = p->n < p->cap ? p->image : NULL; - const char **keys = p->n < p->cap ? p->keys : NULL; - if (img && keys) { - img[p->n] = st->cap.unk || st->tainted - ? NULL - : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); - /* the leaves read: a path that is the prefix of another is dropped */ - const char *json = NULL; + if (p->n < p->cap) { bool whole = true; - for (int i = 0; i < st->nck && whole; i++) { - size_t li = strlen(st->ckeys[i]); - bool prefix = false; - for (int j = 0; j < st->nck && !prefix; j++) { - prefix = j != i && strncmp(st->ckeys[j], st->ckeys[i], li) == 0 && - st->ckeys[j][li] == '.'; - } - if (prefix || !st->ckeys[i][0]) { - continue; - } - char *esc = cbm_arena_alloc(st->a, li * PAIR_LEN + SKIP_ONE); - if (!esc) { - whole = false; - break; - } - size_t o = 0; - for (const char *c = st->ckeys[i]; *c; c++) { - if (*c == '"' || *c == '\\') { - esc[o++] = '\\'; - } - esc[o++] = *c; - } - esc[o] = '\0'; - json = cbm_arena_sprintf(st->a, "%s%s\"%s\"", json ? json : "[", json ? "," : "", esc); - whole = json != NULL; - } - keys[p->n] = json && whole ? cbm_arena_sprintf(st->a, "%s]", json) : NULL; - if (!whole || (json && !keys[p->n])) { - img[p->n] = NULL; /* a mark not written in full: no image, never one unmarked */ - } + const char *keys = hst_capture_keys(st, &whole); + p->image[p->n] = st->cap.unk || st->tainted || !whole + ? NULL /* a mark not written in full: no image, never one unmarked */ + : hst_scalar(st, st->cap.buf ? st->cap.buf : "", st->cap.n); + p->keys[p->n] = whole ? keys : NULL; p->n++; } st->capture = -SKIP_ONE; st->cap = (hsink_t){0}; st->nck = 0; + if (st->ckset) { + cbm_ht_clear(st->ckset); + } } /* Output of the top-level template: into the capture until its newline. */ @@ -2347,25 +2457,6 @@ static const char *he_called_define(hst_t *st, const char *src, TSNode c, bool * return name; } -/* How new a value is: HF_DEEP holds nothing another part holds (fromYaml, - * deepCopy, a dict / list of literals); HF_SHALLOW is a new map or list that - * may hold others' maps by reference (dict / list / omit / pick of values) -- - * a set / unset of it changes nothing else, a merge into it may. */ -typedef enum { HF_NONE = 0, HF_SHALLOW, HF_DEEP } hfresh_lvl_t; - -/* Sprig functions whose result holds nothing another part holds. */ -static bool he_own_fn(const char *s, size_t n) { - static const char *const OWN[] = { - "fromYaml", "fromJson", "fromYamlArray", "fromJsonArray", "deepCopy", "mustDeepCopy", NULL}; - return hy_in(s, n, OWN); -} - -/* Sprig functions whose result is a new map or list of what they are given. */ -static bool he_wrap_fn(const char *s, size_t n) { - static const char *const WRAP[] = {"dict", "list", "omit", "pick", NULL}; - return hy_in(s, n, WRAP); -} - /* The text of call node c's function (a function_call's, or a bare * identifier's); NULL when it has none. */ static const char *he_fn_text(const char *src, TSNode c, size_t *n) { @@ -2378,316 +2469,18 @@ static const char *he_fn_text(const char *src, TSNode c, size_t *n) { return src + b; } -/* The name text of variable node var ("" for `$`). */ -static const char *he_var_text(const char *src, TSNode var, size_t *n) { - TSNode nm = hn_field(var, "name"); - if (ts_node_is_null(nm)) { - *n = 0; - return ""; - } - uint32_t b = ts_node_start_byte(nm); - *n = ts_node_end_byte(nm) - b; - return src + b; +static bool he_is_fn(const char *s, size_t n, const char *name) { + return n == strlen(name) && memcmp(s, name, n) == 0; } -/* The variables a scan saw made new, and how. */ -typedef struct { - const char **name; - size_t *len; - hfresh_lvl_t *lvl; - int n; - int cap; -} hfresh_t; - -static hfresh_lvl_t he_fresh_var(const hfresh_t *f, const char *src, TSNode var) { - size_t vl = 0; - const char *vs = he_var_text(src, var, &vl); - for (int i = f->n - SKIP_ONE; i >= 0; i--) { - if (f->len[i] == vl && memcmp(f->name[i], vs, vl) == 0) { - return f->lvl[i]; - } - } - return HF_NONE; -} - -static bool he_literal(TSNode x) { - return hn_is(x, "interpreted_string_literal") || hn_is(x, "raw_string_literal") || - hn_is(x, "int_literal") || hn_is(x, "float_literal") || hn_is(x, "true") || - hn_is(x, "false") || hn_is(x, "nil"); -} - -/* How new expression `val` is: a constructor's result (`... | fromYaml`, - * `dict`), or a part dug / got / indexed out of a variable that is new. */ -static hfresh_lvl_t he_fresh_value(const hfresh_t *f, const char *src, TSNode val) { - while (hn_is(val, "parenthesized_pipeline") && ts_node_named_child_count(val) == SKIP_ONE) { - val = ts_node_named_child(val, 0); - } - if (hn_is(val, "chained_pipeline")) { - uint32_t c = ts_node_named_child_count(val); - val = c ? ts_node_named_child(val, c - SKIP_ONE) : (TSNode){0}; - } - size_t fl = 0; - const char *fs = - hn_is(val, "function_call") || hn_is(val, "identifier") ? he_fn_text(src, val, &fl) : NULL; - if (!fs) { - return HF_NONE; - } - if (he_own_fn(fs, fl)) { - return HF_DEEP; - } - TSNode args = hn_field(val, "arguments"); - uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); - if (he_wrap_fn(fs, fl)) { - bool literal = true; /* a dict / list of literals holds no one's map */ - for (uint32_t i = 0; i < na && literal; i++) { - literal = he_literal(ts_node_named_child(args, i)); - } - return literal ? HF_DEEP : HF_SHALLOW; - } - if (na == 0) { - return HF_NONE; - } - /* a part of a deep-new value: dig takes its map last; get and index first */ - TSNode from = fl == strlen("dig") && memcmp(fs, "dig", fl) == 0 - ? ts_node_named_child(args, na - SKIP_ONE) - : (fl == strlen("get") && memcmp(fs, "get", fl) == 0) || - (fl == strlen("index") && memcmp(fs, "index", fl) == 0) - ? ts_node_named_child(args, 0) - : (TSNode){0}; - return hn_is(from, "variable") && he_fresh_var(f, src, from) == HF_DEEP ? HF_DEEP : HF_NONE; -} - -/* Is expression x deep-new: a literal, a deep-new variable or value, or a - * dict / list of such? */ -static bool he_deep_value(const hfresh_t *f, const char *src, TSNode x, int depth) { - while (hn_is(x, "parenthesized_pipeline") && ts_node_named_child_count(x) == SKIP_ONE) { - x = ts_node_named_child(x, 0); - } - if (depth > HELM_MAX_NESTING || ts_node_is_null(x)) { - return false; - } - if (he_literal(x)) { - return true; - } - if (hn_is(x, "variable")) { - return he_fresh_var(f, src, x) == HF_DEEP; - } - hfresh_lvl_t lvl = he_fresh_value(f, src, x); - if (lvl != HF_SHALLOW || !hn_is(x, "function_call")) { - return lvl == HF_DEEP; - } - size_t fl = 0; - const char *fs = he_fn_text(src, x, &fl); - if (!fs || !(fl == strlen("dict") || fl == strlen("list")) || - (memcmp(fs, "dict", fl) != 0 && memcmp(fs, "list", fl) != 0)) { - return false; /* omit / pick keep the maps they are given */ - } - TSNode args = hn_field(x, "arguments"); - for (uint32_t i = 0; !ts_node_is_null(args) && i < ts_node_named_child_count(args); i++) { - if (!he_deep_value(f, src, ts_node_named_child(args, i), depth + SKIP_ONE)) { - return false; - } - } - return true; -} - -/* Variable var is `lvl` new from here on (HF_NONE too: what it held before - * no longer counts -- the latest entry of a name is the one looked up). */ -/* Has the scan seen variable var defined or assigned (an entry in f)? */ -static bool he_fresh_seen(const hfresh_t *f, const char *src, TSNode var) { - size_t vl = 0; - const char *vs = he_var_text(src, var, &vl); - for (int i = 0; i < f->n; i++) { - if (f->len[i] == vl && memcmp(f->name[i], vs, vl) == 0) { - return true; - } - } - return false; -} - -/* Note in f that variable var is now lvl-new. A first HF_NONE entry is kept - * too: it hides the running frame's value of var from the scan. */ -static void he_fresh_put(hst_t *st, hfresh_t *f, const char *src, TSNode var, hfresh_lvl_t lvl) { - if (ts_node_is_null(var) || - (lvl == HF_NONE && he_fresh_seen(f, src, var) && he_fresh_var(f, src, var) == HF_NONE)) { - return; - } - if (f->n == f->cap) { - int cap = f->cap ? f->cap * PAIR_LEN : CBM_SZ_16; - const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); - size_t *gl = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gl)); - hfresh_lvl_t *gv = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*gv)); - if (!g || !gl || !gv) { - return; - } - if (f->n) { - memcpy(g, f->name, (size_t)f->n * sizeof(*g)); - memcpy(gl, f->len, (size_t)f->n * sizeof(*gl)); - memcpy(gv, f->lvl, (size_t)f->n * sizeof(*gv)); - } - *f = (hfresh_t){g, gl, gv, f->n, cap}; - } - f->name[f->n] = he_var_text(src, var, &f->len[f->n]); - f->lvl[f->n] = lvl; - f->n++; -} - -/* Is expression x deep-new for the scan (f) -- or, a variable the scan did - * not see made, with fr (a part of the running template), a value of fr - * that is own? */ -static bool he_value_owned(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode x) { - if (he_deep_value(f, src, x, 0)) { - return true; - } - if (!fr || !hn_is(x, "variable") || he_fresh_seen(f, src, x)) { - return false; - } - size_t l = 0; - const char *name = hn_var_name(fr, x, &l); - hvar_t *v = hst_var(st, fr, name, l); - return v && v->v.own; -} - -/* Is call node c a set / unset / merge of variable var whose every other - * argument is own (the variable stays as new as it is)? */ -static bool he_self_change(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode c, - TSNode var) { - size_t fl = 0; - const char *fs = hn_is(c, "function_call") ? he_fn_text(src, c, &fl) : NULL; - TSNode args = fs && he_mutator(fs, fl) ? hn_field(c, "arguments") : (TSNode){0}; - uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); - TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; - if (!hn_is(first, "variable")) { - return false; - } - size_t al = 0; - size_t bl = 0; - const char *as = he_var_text(src, first, &al); - const char *bs = he_var_text(src, var, &bl); - if (al != bl || memcmp(as, bs, al) != 0) { - return false; - } - for (uint32_t i = SKIP_ONE; i < na; i++) { - if (!he_value_owned(st, fr, f, src, ts_node_named_child(args, i))) { - return false; - } - } - return true; -} - -/* A definition (`$x := v`) or an assignment (`$x = v`) into f: how new $x is - * now -- what v is, or, for `$x = set $x ...` of own values, what $x was. */ -static void he_fresh_note(hst_t *st, hframe_t *fr, hfresh_t *f, const char *src, TSNode def) { - TSNode var = hn_field(def, "variable"); - TSNode val = hn_field(def, "value"); - if (ts_node_is_null(var)) { - return; - } - hfresh_lvl_t lvl = he_deep_value(f, src, val, 0) ? HF_DEEP : he_fresh_value(f, src, val); - if (lvl == HF_NONE && hn_is(def, "assignment") && he_self_change(st, fr, f, src, val, var)) { - lvl = he_fresh_var(f, src, var); - } - he_fresh_put(st, f, src, var, lvl); -} - -/* Are the values set / merge call c puts into its target own (see - * he_value_owned)? */ -static bool he_puts_deep(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode c) { - TSNode args = hn_field(c, "arguments"); - uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); - for (uint32_t i = SKIP_ONE; i < na; i++) { - if (!he_value_owned(st, fr, f, src, ts_node_named_child(args, i))) { - return false; - } - } - return !hn_is(ts_node_parent(c), "chained_pipeline"); /* a piped value: not known here */ -} - -/* Does expression x read `.` (a field of it)? In a with / range body `.` is - * not the frame's. */ -static bool he_reads_dot(TSNode x) { - TSTreeCursor cur = ts_tree_cursor_new(x); - bool found = false; - for (bool more = true; more && !found;) { - TSNode c = ts_tree_cursor_current_node(&cur); - found = hn_is(c, "dot") || hn_is(c, "field"); - if (found || ts_tree_cursor_goto_first_child(&cur)) { - continue; - } - while (!ts_tree_cursor_goto_next_sibling(&cur)) { - if (!ts_tree_cursor_goto_parent(&cur)) { - more = false; - break; - } - } - } - ts_tree_cursor_delete(&cur); - return found; -} - -/* Does expression x name a variable the scan (f) saw defined or assigned -- - * one whose value in the running frame is not the one x would read? */ -static bool he_names_rebound(const hfresh_t *f, const char *src, TSNode x) { - TSTreeCursor cur = ts_tree_cursor_new(x); - bool found = false; - for (bool more = true; more && !found;) { - TSNode c = ts_tree_cursor_current_node(&cur); - found = hn_is(c, "variable") && he_fresh_seen(f, src, c); - if (found || ts_tree_cursor_goto_first_child(&cur)) { - continue; - } - while (!ts_tree_cursor_goto_next_sibling(&cur)) { - if (!ts_tree_cursor_goto_parent(&cur)) { - more = false; - break; - } - } - } - ts_tree_cursor_delete(&cur); - return found; -} - -/* What a change of the value of expression x (in frame fr; NULL: not - * running) would reach: false when it reaches nothing another part holds -- - * x is own, or a known map, now poisoned (`deep`: a merge). A value not known - * here -- or x naming a variable the part (f) redefined: true. */ -static bool he_change_reaches(hst_t *st, hframe_t *fr, const hfresh_t *f, const char *src, TSNode x, - bool dot_known, bool deep) { - if (!fr || ts_node_is_null(x) || (!dot_known && he_reads_dot(x)) || - he_names_rebound(f, src, x)) { - return true; - } - /* read without recording: the part is not run, its reads are no capture's */ - int capture = st->capture; - st->capture = -SKIP_ONE; - hv_t v = he_eval(st, fr, x); - st->capture = capture; - /* a known map is poisoned even when own: what it holds is not known now */ - bool poisoned = hst_poison(st, &v, deep); - return !poisoned && !v.own; -} - -static void he_revoke_own(hst_t *st, hframe_t *fr, TSNode x); - -/* May running node n (of text src) change a map another part holds -- a set / - * unset / merge whose target is not a value n made new itself (`$d := dict`, - * `set (dict) ...`; a merge needs it deep-new), in n or in a define it - * includes (whose changes reach only its argument)? With fr (n is a part of - * the running template not run), a target or include argument -- `.`, a - * field, a call's result such as `index .Values "image"`, a variable of fr -- - * is looked at: own, or a known map (then poisoned): no. A change that puts a - * value another part may hold into a variable makes it no longer new / own. - * An include of a computed name is a template file by path, another - * template's work, which no render models. The text a tpl renders is not - * looked into. `made` (NULL: none) is what the parts before n in its body - * made new or redefined; n's own are added to it. */ -static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hframe_t *fr, - bool dot_known, hfresh_t *made) { +/* May running node n (of text src) change a map in place? It may when it + * holds a set / unset / merge (whatever the target), a tpl (its text is not + * looked into), or an include / template of a name not known or of one that + * may (he_define_may_mutate). A variable's name ($set) is no call. */ +static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth) { if (depth > HELM_MAX_NESTING) { return true; } - hfresh_t own = {0}; /* variables n makes from a new map */ - hfresh_t *fresh = made ? made : &own; TSTreeCursor cur = ts_tree_cursor_new(n); bool found = false; bool more = true; @@ -2696,51 +2489,17 @@ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hfram const char *t = ts_node_type(c); size_t fl = 0; const char *fs = NULL; - if (strcmp(t, "variable_definition") == 0 || strcmp(t, "assignment") == 0) { - he_fresh_note(st, fr, fresh, src, c); - } else if (strcmp(t, "function_call") == 0 && (fs = he_fn_text(src, c, &fl)) != NULL && - he_mutator(fs, fl)) { - TSNode args = hn_field(c, "arguments"); - TSNode target = ts_node_is_null(args) ? (TSNode){0} : ts_node_named_child(args, 0); - bool deep = fs[0] == 'm'; /* a merge reaches the maps within its target too */ - bool puts_deep = he_puts_deep(st, fr, fresh, src, c); - /* how new the target is: a variable n made, or a new value itself - * (`set (dict) ...`); anything else -- `.`, a field, a call's - * result (`index .Values "image"`) -- is looked at in fr */ - hfresh_lvl_t lvl = hn_is(target, "variable") ? he_fresh_var(fresh, src, target) - : ts_node_is_null(target) ? HF_NONE - : he_deep_value(fresh, src, target, 0) - ? HF_DEEP - : he_fresh_value(fresh, src, target); - bool spared = lvl == HF_DEEP || (lvl == HF_SHALLOW && !deep); - found = !spared && he_change_reaches(st, fr, fresh, src, target, dot_known, deep); - if (!puts_deep && hn_is(target, "variable")) { - /* it now holds a value another part may hold */ - if (lvl == HF_DEEP) { - he_fresh_put(st, fresh, src, target, HF_SHALLOW); - } else if (fr && lvl == HF_NONE) { - he_revoke_own(st, fr, target); - } - } + bool call = strcmp(t, "function_call") == 0 || + (strcmp(t, "identifier") == 0 && hn_is(ts_node_parent(c), "chained_pipeline")); + if (call && (fs = he_fn_text(src, c, &fl)) != NULL && + (he_mutator(fs, fl) || he_is_fn(fs, fl, "tpl"))) { + found = true; } else if (strcmp(t, "function_call") == 0 || strcmp(t, "template_action") == 0 || strcmp(t, "block_action") == 0) { bool computed = false; const char *name = he_called_define(st, src, c, &computed); - if (!computed && name && he_define_may_mutate(st, name, depth + SKIP_ONE)) { - /* what the define changes is its argument (or a map within it) */ - TSNode args = hn_field(c, "arguments"); - TSNode arg = hn_is(c, "function_call") - ? (ts_node_is_null(args) ? (TSNode){0} - : ts_node_named_child(args, SKIP_ONE)) - : hn_field(c, "argument"); - found = !he_deep_value(fresh, src, arg, 0) && - he_change_reaches(st, fr, fresh, src, arg, dot_known, true); - } - } else if (strcmp(t, "identifier") == 0 && (fs = he_fn_text(src, c, &fl)) != NULL && - he_mutator(fs, fl) && hn_is(ts_node_parent(c), "chained_pipeline")) { - found = true; /* `x | set`: the target is the piped value, not known here */ + found = computed || (name && he_define_may_mutate(st, name, depth + SKIP_ONE)); } - /* a variable's name ($set) is no call */ if (found || (strcmp(t, "variable") != 0 && ts_tree_cursor_goto_first_child(&cur))) { continue; } @@ -2755,6 +2514,51 @@ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth, hfram return found || st->stop; } +/* The chart's template file an include names -- Helm names each one + * `<.Template.BasePath>/` -- or NULL: none of this chart's. */ +static const hd_file_t *he_template_file(hst_t *st, const char *name) { + if (!st->base_path || !st->chart_dir) { + return NULL; + } + size_t bl = strlen(st->base_path); + if (strncmp(name, st->base_path, bl) != 0 || name[bl] != '/') { + return NULL; + } + const char *rel = cbm_arena_sprintf(st->a, "%s%stemplates/%s", st->chart_dir, + st->chart_dir[0] ? "/" : "", name + bl + SKIP_ONE); + return rel ? hd_file_find(st->defs, rel) : NULL; +} + +/* May template file f, included by its name and so run whole, change a map + * in place (he_may_mutate over all of it)? Memoized; a file reached again + * while it is being scanned -- one including itself -- may. A no that leaned + * on a define still being scanned is not kept. */ +static bool he_file_may_mutate(hst_t *st, const hd_file_t *f, int depth) { + enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3 }; + if (!f->tree) { + return true; + } + if (!st->file_mutates) { + st->file_mutates = cbm_ht_create(0); + } + if (!st->file_mutates) { + return true; + } + uintptr_t memo = (uintptr_t)cbm_ht_get(st->file_mutates, f->rel); + if (memo) { + return memo != HM_NO; + } + cbm_ht_set(st->file_mutates, f->rel, (void *)(uintptr_t)HM_BUSY); + long hits = st->busy_hits; + bool m = he_may_mutate(st, f->src, ts_tree_root_node(f->tree), depth); + if (m || st->busy_hits == hits) { + cbm_ht_set(st->file_mutates, f->rel, (void *)(uintptr_t)(m ? HM_YES : HM_NO)); + } else { + (void)cbm_ht_delete(st->file_mutates, f->rel); + } + return m; +} + /* Memoized per define. A define reached again while it is being scanned * counts as no for now; a no that leaned on such a one is not kept (STALE: * scanned again when next asked), so defines including each other never fix @@ -2763,7 +2567,10 @@ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3, HM_STALE = 4 }; const hd_def_t *d = hd_find(st->defs, name); if (!d) { - return false; /* a define not found is an error: nothing of it runs */ + /* a template file of the chart by its name (the checksum idiom) is + * looked into; a name not the chart's (a library's) may */ + const hd_file_t *f = he_template_file(st, name); + return f ? he_file_may_mutate(st, f, depth) : true; } if (!st->mutates) { st->mutates = cbm_ht_create(0); @@ -2781,7 +2588,7 @@ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { cbm_ht_set(st->mutates, key, (void *)(uintptr_t)HM_BUSY); } long hits = st->busy_hits; - bool m = he_may_mutate(st, d->file->src, d->node, depth, NULL, false, NULL); + bool m = he_may_mutate(st, d->file->src, d->node, depth); bool leaned = st->busy_hits > hits; /* on a define still being scanned */ if (st->mutates && key) { uintptr_t res = m ? HM_YES : leaned ? HM_STALE : HM_NO; @@ -2790,116 +2597,17 @@ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { return m; } -static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n); - -/* A define not run would change its argument v (or a map within it): a known - * one is poisoned whole; one not known that is not own spoils what follows. */ -static void he_spoil_arg(hst_t *st, const hv_t *v) { - bool known = hv_is_map(v) || hv_is_list(v); - if (!(known && hst_poison(st, v, true)) && !v->own && - (v->k == HV_UNK || hv_is_map(v) || hv_is_list(v))) { - st->tainted = true; - } -} - -/* Is the result of `fn` over a own (see hv_t): a part dug / got / indexed / - * taken out of an own value no change has touched (a poisoned map may hold - * another's now); a dict / list / default / coalesce / ternary whose every map - * or list (and every value not known) is own? */ -static bool he_own_result(const hst_t *st, const char *fn, const hv_t *a, int n) { - if (n < SKIP_ONE) { - return strcmp(fn, "dict") == 0 || strcmp(fn, "list") == 0; /* empty and new */ - } - if (strcmp(fn, "dig") == 0) { - return a[n - SKIP_ONE].own && !hv_poisoned(st, &a[n - SKIP_ONE]); - } - if (strcmp(fn, "get") == 0 || strcmp(fn, "index") == 0 || strcmp(fn, "first") == 0 || - strcmp(fn, "last") == 0) { - return a[0].own && !hv_poisoned(st, &a[0]); - } - if (strcmp(fn, "dict") == 0 || strcmp(fn, "list") == 0 || strcmp(fn, "default") == 0 || - strcmp(fn, "coalesce") == 0 || strcmp(fn, "ternary") == 0) { - for (int i = 0; i < n; i++) { - if (!a[i].own && (a[i].k == HV_UNK || hv_is_map(&a[i]) || hv_is_list(&a[i]))) { - return false; - } - } - return true; - } - return false; -} - -/* A function call `fn` with its evaluated arguments (the piped value last), - * its result marked own when it is. */ -static hv_t he_apply(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { - if (he_own_fn(fn, strlen(fn))) { - return HV_UNKNOWN_OWN; /* fromYaml / deepCopy: a new value, known or not */ - } - hv_t r = he_apply_fn(st, fr, fn, a, n); - r.own = r.own || he_own_result(st, fn, a, n); - return r; -} - -/* The variable expression x reads its value out of (`$v`, `$v.a`, `(index $v - * "a")`, `get $v "a"`, `dig "a" $v`); a null node when there is none. */ -static TSNode he_root_var(const hframe_t *fr, TSNode x) { - for (int depth = 0; depth < HELM_MAX_NESTING && !ts_node_is_null(x); depth++) { - if (hn_is(x, "variable")) { - return x; - } - if (hn_is(x, "parenthesized_pipeline") && ts_node_named_child_count(x) == SKIP_ONE) { - x = ts_node_named_child(x, 0); - } else if (hn_is(x, "selector_expression")) { - x = hn_field(x, "operand"); - } else if (hn_is(x, "function_call")) { - size_t fl = 0; - const char *fs = he_fn_text(fr->src, x, &fl); - TSNode args = hn_field(x, "arguments"); - uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); - bool last = fs && fl == strlen("dig") && memcmp(fs, "dig", fl) == 0; - bool first = fs && ((fl == strlen("get") && memcmp(fs, "get", fl) == 0) || - (fl == strlen("index") && memcmp(fs, "index", fl) == 0) || - (fl == strlen("first") && memcmp(fs, "first", fl) == 0) || - (fl == strlen("last") && memcmp(fs, "last", fl) == 0)); - if (!na || (!first && !last)) { - break; - } - x = ts_node_named_child(args, last ? na - SKIP_ONE : 0); - } else { - break; - } - } - return (TSNode){0}; -} - -/* A change put a value another part may hold into what expression x reads: - * the variable it reads it out of is own no more. */ -static void he_revoke_own(hst_t *st, hframe_t *fr, TSNode x) { - TSNode var = he_root_var(fr, x); - if (ts_node_is_null(var)) { - return; - } - size_t l = 0; - const char *name = hn_var_name(fr, var, &l); - hvar_t *v = hst_var(st, fr, name, l); - if (v) { - v->v.own = false; - } -} - static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) { for (int i = 0; i < n; i++) { if (a[i].k == HV_UNK || hv_poisoned(st, &a[i])) { - /* an include of a known define not run (an argument UNKNOWN or - * poisoned): a change it would make to its argument is missed -- - * a known argument is poisoned whole, one not known and not own - * spoils what follows (one of a name not known -- a template file - * by path -- is another template's work, which no render models) */ - if (strcmp(fn, "include") == 0 && a[0].k == HV_STR && n == PAIR_LEN) { - const char *dn = cbm_arena_strndup(st->a, a[0].s, a[0].sn); - if (!dn || he_define_may_mutate(st, dn, st->nesting)) { - he_spoil_arg(st, &a[1]); - } + /* an include not run (an argument UNKNOWN or poisoned) that may + * change maps, or a tpl not run: what it would change is missed */ + bool runs_code = strcmp(fn, "include") == 0 || strcmp(fn, "tpl") == 0; + const char *dn = strcmp(fn, "include") == 0 && a[0].k == HV_STR + ? cbm_arena_strndup(st->a, a[0].s, a[0].sn) + : NULL; + if (runs_code && (!dn || he_define_may_mutate(st, dn, st->nesting))) { + hst_taint(st, fn); } return HV_UNKNOWN; } @@ -2912,6 +2620,7 @@ static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) } if (strcmp(fn, "tpl") == 0 && n == PAIR_LEN && a[0].k == HV_STR) { if (!st->parser) { + hst_taint(st, "tpl"); return HV_UNKNOWN; } char *src = cbm_arena_strndup(st->a, a[0].s, a[0].sn); @@ -2936,6 +2645,7 @@ static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) TSNode root = ts_tree_root_node(t); if (ts_node_has_error(root) || ++st->nesting > HELM_MAX_NESTING) { st->nesting -= ts_node_has_error(root) ? 0 : SKIP_ONE; + hst_taint(st, "tpl"); return HV_UNKNOWN; } hsink_t out = {0}; @@ -3177,7 +2887,7 @@ static hv_t he_apply_fn(hst_t *st, hframe_t *fr, const char *fn, hv_t *a, int n) lines += s.s[i] == '\n'; } if ((uint64_t)w > HELM_MAX_OUTPUT / lines) { - st->stop = true; + hst_trip(st, "output"); return HV_UNKNOWN; } hsink_t out = {0}; @@ -3256,29 +2966,21 @@ static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { TSNode args = hn_is(call, "identifier") ? (TSNode){0} : hn_field(call, "arguments"); uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); if (he_mutator(fn, fl)) { - /* set / unset / merge change a map in place -- not modelled: what it - * holds is UNKNOWN from here on (after a merge, every map within it). - * A target not known that is not own -- whatever expression gives it: - * index / get / first return a map another part holds -- spoils every - * image after it. */ + /* set / unset / merge change a map in place -- not run: the map is + * poisoned, what it holds UNKNOWN from here on (after a merge, every + * map within it too). That is exact only for a known map (a merge: + * whose every map within is known, see hst_poison); any other target + * -- one not known, whatever gives it -- leaves every image after it + * UNKNOWN. */ TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; hv_t target = na ? he_eval(st, fr, first) : piped ? *piped : HV_UNKNOWN; - /* what goes into it: own too, or the map now holds another's */ - bool put_own = true; - for (uint32_t i = SKIP_ONE; i < na + (piped && na ? SKIP_ONE : 0); i++) { - hv_t v = i < na ? he_eval(st, fr, ts_node_named_child(args, i)) : *piped; - put_own = put_own && (v.own || (v.k != HV_UNK && !hv_is_map(&v) && !hv_is_list(&v))); + for (uint32_t i = SKIP_ONE; i < na; i++) { + (void)he_eval(st, fr, ts_node_named_child(args, i)); /* what it runs */ } - /* a known map is poisoned even when own: what it holds is not known now */ - bool poisoned = hst_poison(st, &target, fn[0] == 'm'); - if (!poisoned && !target.own) { - st->tainted = true; + if (!hv_is_map(&target) || !hst_poison(st, &target, fn[0] == 'm')) { + hst_taint(st, fn); } - if (!put_own && !ts_node_is_null(first)) { - he_revoke_own(st, fr, first); - } - /* the changed map is what set / merge return */ - return target.own && put_own ? HV_UNKNOWN_OWN : HV_UNKNOWN; + return HV_UNKNOWN; /* the changed map */ } bool lazy = strcmp(fn, "and") == 0 || strcmp(fn, "or") == 0; if (lazy) { @@ -3307,7 +3009,7 @@ static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { if (piped) { a[na] = *piped; } - return he_apply(st, fr, fn, a, n); + return he_apply_fn(st, fr, fn, a, n); } static hv_t he_literal_int(hframe_t *fr, TSNode n) { @@ -3463,77 +3165,28 @@ static void he_items(hst_t *st, hframe_t *fr, TSNode parent, const char *field) } } -/* Does assigning expression `value` to variable var keep it own (when it is): - * a deep-new value, or a set / merge of var itself with own values - * (`$dst = ... | fromYaml | merge $dst`)? */ -static bool he_assign_owns(hst_t *st, hframe_t *fr, const hfresh_t *f, TSNode var, TSNode value) { - const char *src = fr->src; - if (he_deep_value(f, src, value, 0)) { - return true; - } - while (hn_is(value, "parenthesized_pipeline") && ts_node_named_child_count(value) == SKIP_ONE) { - value = ts_node_named_child(value, 0); - } - TSNode call = value; - TSNode piped = {0}; - if (hn_is(value, "chained_pipeline")) { - uint32_t c = ts_node_named_child_count(value); - if (c < PAIR_LEN) { - return false; - } - call = ts_node_named_child(value, c - SKIP_ONE); - piped = ts_node_named_child(value, c - PAIR_LEN); - } - size_t fl = 0; - const char *fs = hn_is(call, "function_call") ? he_fn_text(src, call, &fl) : NULL; - TSNode args = fs && he_mutator(fs, fl) ? hn_field(call, "arguments") : (TSNode){0}; - uint32_t na = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); - TSNode first = na ? ts_node_named_child(args, 0) : (TSNode){0}; - if (!hn_is(first, "variable")) { - return false; - } - size_t al = 0; - size_t bl = 0; - const char *as = he_var_text(src, first, &al); - const char *bs = he_var_text(src, var, &bl); - if (al != bl || memcmp(as, bs, al) != 0) { - return false; - } - for (uint32_t i = SKIP_ONE; i < na; i++) { - if (!he_value_owned(st, fr, f, src, ts_node_named_child(args, i))) { - return false; - } - } - return ts_node_is_null(piped) || he_value_owned(st, fr, f, src, piped); -} - /* How a part not run would leave the loop around it. */ enum { HL_NONE = 0, HL_CONTINUE = 1, HL_BREAK = 2 }; /* The variables of the frames around n that n assigns (`$x = ...`) become - * UNKNOWN (an own one stays own when what n gives it keeps it so); returns - * how n would leave the loop around it (HL_*) -- by a break / continue not - * inside a range of n's own (`in_range`: n is a range's body). `fresh` is - * what the parts before n in its body made new; n's own are added to it. */ -static int he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range, hfresh_t *fresh) { + * UNKNOWN; returns how n would leave the loop around it (HL_*) -- by a break + * / continue not inside a range of n's own (`in_range`: n is a range's + * body). */ +static int he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { TSTreeCursor cur = ts_tree_cursor_new(n); int leave = HL_NONE; bool more = true; while (more && hst_step(st)) { TSNode c = ts_tree_cursor_current_node(&cur); const char *t = ts_node_type(c); - if (strcmp(t, "variable_definition") == 0) { - he_fresh_note(st, fr, fresh, fr->src, c); - } else if (strcmp(t, "assignment") == 0) { + if (strcmp(t, "assignment") == 0) { TSNode var = hn_field(c, "variable"); size_t l = 0; const char *name = ts_node_is_null(var) ? NULL : hn_var_name(fr, var, &l); hvar_t *v = name ? hst_var(st, fr, name, l) : NULL; if (v) { - bool keeps = v->v.own && he_assign_owns(st, fr, fresh, var, hn_field(c, "value")); - v->v = keeps ? HV_UNKNOWN_OWN : HV_UNKNOWN; + v->v = HV_UNKNOWN; } - he_fresh_note(st, fr, fresh, fr->src, c); } else if (strcmp(t, "break_action") == 0 || strcmp(t, "continue_action") == 0) { /* a range around it inside n (n itself included) is the one it leaves */ bool own = in_range || hn_is(n, "range_action"); @@ -3560,38 +3213,26 @@ static int he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range, hfr } /* The children of n but its condition / collection (what was run), each: - * the variables it would assign and the changes it would make spoiled (see - * he_scan_skipped / he_may_mutate); returns how it would leave the loop. */ -static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, const char *only, bool in_range, - bool dot_known) { + * the variables it would assign UNKNOWN, and a change of maps it may make + * (he_may_mutate) leaves every image after it UNKNOWN; returns how it would + * leave the loop. */ +static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, const char *only, bool in_range) { int leave = HL_NONE; bool mutates = false; - /* what a branch's parts made new so far: a later part of it sees it */ - hfresh_t scanned = {0}; - hfresh_t made = {0}; - const char *branch = NULL; uint32_t cc = ts_node_child_count(n); for (uint32_t i = 0; i < cc; i++) { TSNode c = ts_node_child(n, i); const char *f = ts_node_field_name_for_child(n, i); - if (f && strcmp(f, "condition") == 0) { - branch = NULL; /* an else if: a new branch */ - } if (!ts_node_is_named(c) || hn_is(c, "range_variable_definition") || (f && (strcmp(f, "condition") == 0 || strcmp(f, "range") == 0)) || (only && !(f && strcmp(f, only) == 0))) { continue; } - if (f && (!branch || strcmp(f, branch) != 0)) { - branch = f; - scanned = (hfresh_t){0}; - made = (hfresh_t){0}; - } - leave |= he_scan_skipped(st, fr, c, in_range, &scanned); - mutates = mutates || he_may_mutate(st, fr->src, c, st->nesting, fr, dot_known, &made); + leave |= he_scan_skipped(st, fr, c, in_range); + mutates = mutates || he_may_mutate(st, fr->src, c, st->nesting); } if (mutates) { - st->tainted = true; + hst_taint(st, "skipped"); } return leave; } @@ -3605,8 +3246,7 @@ static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, const char *only, b static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { hst_out_unknown(st, fr); bool range = hn_is(n, "range_action"); - bool with = hn_is(n, "with_action"); - int leave = he_spoil_parts(st, fr, n, NULL, range, !range && !with); + int leave = he_spoil_parts(st, fr, n, NULL, range); if (leave == HL_NONE) { return; } @@ -3617,7 +3257,7 @@ static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { loop = ts_node_parent(loop); } if (hn_is(loop, "range_action")) { - (void)he_spoil_parts(st, fr, loop, "body", true, false); + (void)he_spoil_parts(st, fr, loop, "body", true); } if (leave & HL_BREAK) { st->brk = true; @@ -3764,10 +3404,9 @@ static void he_template(hst_t *st, hframe_t *fr, TSNode n) { hv_t a = ts_node_is_null(arg) ? hv_nil() : he_eval(st, fr, arg); if (a.k == HV_UNK || hv_poisoned(st, &a)) { hst_out_unknown(st, fr); - /* the define not run: a change it would make to its argument is - * missed (see he_spoil_arg) */ + /* the define not run: a change of maps it may make is missed */ if (he_define_may_mutate(st, name, st->nesting)) { - he_spoil_arg(st, &a); + hst_taint(st, "template"); } return; } @@ -3819,8 +3458,18 @@ static void he_item(hst_t *st, hframe_t *fr, TSNode c) { /* Define `name` rendered with dot / $ = arg into `into`. */ static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *into) { const hd_def_t *d = hd_find(st->defs, name); - if (!d || ++st->nesting > HELM_MAX_NESTING) { - st->nesting -= d ? SKIP_ONE : 0; + if (!d) { + /* not run -- a template file of the chart by its name (the checksum + * idiom) or a name not the chart's: its output is UNKNOWN, and a + * change it may make (he_define_may_mutate) is missed */ + if (he_define_may_mutate(st, name, st->nesting)) { + hst_taint(st, "include"); + } + return HV_UNKNOWN; + } + if (++st->nesting > HELM_MAX_NESTING) { + st->nesting--; + hst_taint(st, "include"); /* nested too deep: not run */ return HV_UNKNOWN; } int mark = st->nvars; @@ -3862,6 +3511,32 @@ static hv_t hv_cstr(const char *s) { return s ? hv_str(s, strlen(s)) : hv_nil(); } +/* The probes' captures, made in the render's own arena, copied into a: the + * caller keeps a render's results, never its work. An image whose mark is not + * copied is not kept either. */ +static void helm_keep_results(CBMArena *a, cbm_helm_probe_t *probes, int nprobes) { + for (int i = 0; i < nprobes; i++) { + cbm_helm_probe_t *p = &probes[i]; + int n = p->n; + const char **img = n ? cbm_arena_alloc(a, (size_t)n * sizeof(*img)) : NULL; + const char **keys = n ? cbm_arena_alloc(a, (size_t)n * sizeof(*keys)) : NULL; + if (!img || !keys) { + n = 0; + img = NULL; + keys = NULL; + } + for (int c = 0; c < n; c++) { + const char *k = p->keys[c] ? cbm_arena_strdup(a, p->keys[c]) : NULL; + img[c] = p->image[c] && (k || !p->keys[c]) ? cbm_arena_strdup(a, p->image[c]) : NULL; + keys[c] = k; + } + p->image = img; + p->keys = keys; + p->n = n; + p->cap = n; + } +} + bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes) { @@ -3874,14 +3549,21 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, if (!src || nprobes == 0) { return true; } - hst_t st = {.a = a, + /* the render's work lives in its own arena, freed when it ends: the + * memory budget bounds one render, and no render keeps another's */ + CBMArena work; + cbm_arena_init_lazy(&work, CBM_SZ_4K); + hst_t st = {.a = &work, .defs = defs, .top_src = src, .top_len = len, .probes = probes, .nprobes = nprobes, + .rel = rel, + .base_path = chart ? chart->base_path : NULL, + .chart_dir = chart ? chart->dir : NULL, .capture = -SKIP_ONE, - .arena0 = cbm_arena_total(a)}; + .arena0 = cbm_arena_total(&work)}; st.scratch = yyjson_mut_doc_new(cbm_json_writer_allocator()); st.parser = ts_parser_new(); if (!st.scratch || !st.parser || @@ -3892,13 +3574,14 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, if (st.scratch) { yyjson_mut_doc_free(st.scratch); } + cbm_arena_destroy(&work); return true; } int nl = SKIP_ONE; for (size_t i = 0; i < len; i++) { nl += src[i] == '\n'; } - uint32_t *lines = cbm_arena_alloc(a, (size_t)nl * sizeof(*lines)); + uint32_t *lines = cbm_arena_alloc(&work, (size_t)nl * sizeof(*lines)); TSTree *tree = lines ? ts_parser_parse_string(st.parser, NULL, src, (uint32_t)len) : NULL; if (tree) { int k = 0; @@ -3913,7 +3596,8 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, yyjson_mut_val *vals = values ? values : yyjson_mut_obj(st.scratch); hv_map_t *ch = hv_struct(&st, 3); hv_map_t *rl = hv_struct(&st, SKIP_ONE); - hv_map_t *root = hv_struct(&st, PAIR_LEN + SKIP_ONE); + hv_map_t *tp = st.base_path ? hv_struct(&st, SKIP_ONE) : NULL; + hv_map_t *root = hv_struct(&st, PAIR_LEN + PAIR_LEN); if (ch && rl && root) { hv_struct_set(ch, "Name", hv_cstr(chart ? chart->name : NULL)); hv_struct_set(ch, "Version", hv_cstr(chart ? chart->version : NULL)); @@ -3922,6 +3606,11 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, hv_struct_set(root, "Values", hv_json(vals, "")); hv_struct_set(root, "Chart", (hv_t){.k = HV_MAP, .m = ch}); hv_struct_set(root, "Release", (hv_t){.k = HV_MAP, .m = rl}); + if (tp) { + /* .Template.Name is not modelled: UNKNOWN */ + hv_struct_set(tp, "BasePath", hv_cstr(st.base_path)); + hv_struct_set(root, "Template", (hv_t){.k = HV_MAP, .m = tp}); + } hv_t dot = {.k = HV_MAP, .m = root}; hframe_t fr = {.src = src, .dot = dot, .var_base = 0, .out = NULL}; hst_var_push(&st, "", 0, dot); @@ -3938,12 +3627,18 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, if (st.mutates) { cbm_ht_free(st.mutates); } + if (st.file_mutates) { + cbm_ht_free(st.file_mutates); + } if (st.poison_maps) { cbm_ht_free(st.poison_maps); } if (st.poison_paths) { cbm_ht_free(st.poison_paths); } + if (st.ckset) { + cbm_ht_free(st.ckset); + } if (st.stop) { /* Helm would not finish this template: none of its images is known, * also those captured before the budget ran out */ @@ -3954,9 +3649,9 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, } } cbm_log_warn("helm.render.budget", "file", rel ? rel : "", "reason", - st.steps > HELM_MAX_STEPS ? "steps" - : st.written >= HELM_MAX_OUTPUT ? "output" - : "memory"); + st.why ? st.why : "memory"); } + helm_keep_results(a, probes, nprobes); + cbm_arena_destroy(&work); return !st.stop; } diff --git a/src/pipeline/helm_template.h b/src/pipeline/helm_template.h index 44985a245..e2136b5e8 100644 --- a/src/pipeline/helm_template.h +++ b/src/pipeline/helm_template.h @@ -44,11 +44,17 @@ bool cbm_helm_values_set(yyjson_mut_doc *doc, yyjson_mut_val *root, const char * * 1.1) into out; false for a map, a list or null. */ bool cbm_helm_scalar_text(yyjson_mut_val *v, char *out, size_t sz); -/* A chart's metadata, as its templates see it under .Chart. */ +/* A chart's metadata, as its templates see it under .Chart -- and where its + * templates are: .Template.BasePath as Helm sets it (`/templates`, + * the chart path its root's name and `/charts/` per subchart + * below it; NULL: not known) and the chart's directory as its template files + * are added to the defines ("" at the repository root). */ typedef struct { const char *name; const char *version; const char *app_version; + const char *base_path; + const char *dir; } cbm_helm_chart_meta_t; /* The named templates (`define` / `block`) of one chart family -- a chart and @@ -75,10 +81,11 @@ typedef struct { /* Render template `rel` (text src, len) with `values` (its .Values; NULL: * an empty map), `chart` and the family's defines, filling `probes` (strings - * in arena a). Values paths are dot-joined keys, a key's own `.` escaped - * (`\.`), as --set writes them. False when the render ran past its step or - * output budget -- a template Helm would not finish either (logged): then - * every capture is UNKNOWN. */ + * in arena a, which gets only these: the render works in an arena of its own). + * Values paths are dot-joined keys, a key's own `.` escaped (`\.`), as --set + * writes them. False when the render ran past its step, output or memory + * budget -- a template Helm would not finish either (logged): then every + * capture is UNKNOWN. */ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, yyjson_mut_val *values, const cbm_helm_chart_meta_t *chart, const cbm_helm_defines_t *defs, cbm_helm_probe_t *probes, int nprobes); diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index d20ca4cce..034e8984c 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -3920,9 +3920,6 @@ typedef struct { int tcap; k8s_himgs_t dflt; /* the containers' images with the defaults */ int edges; - /* the release rendered: its set_sensitive names (JSON array; NULL: none) -- - * an image reading one is not known, its value never read */ - const char *sensitive; } k8s_helm_t; /* The templated images of the facts, kept past them. */ @@ -4319,9 +4316,9 @@ static void k8s_helm_take(cbm_pipeline_ctx_t *ctx, const cbm_helm_probe_t *p, in /* The images of templated containers [from, to) of one template, rendered * with `values` (NULL: the chart's own). */ static void k8s_helm_render(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, - const k8s_hchart_t *c, const char *as_name, const k8s_timg_t *t, - int from, int to, yyjson_mut_val *values, const char *src, size_t len, - cbm_helm_probe_t *probes, int np) { + const k8s_hchart_t *c, const char *as_name, const char *full, + const k8s_timg_t *t, int from, int to, yyjson_mut_val *values, + const char *src, size_t len, cbm_helm_probe_t *probes, int np) { char fam[CBM_SZ_1K]; k8s_chart_family(b, c->dir, fam, sizeof(fam)); const cbm_helm_defines_t *defs = k8s_helm_defines(ctx, b, h, fam); @@ -4329,6 +4326,9 @@ static void k8s_helm_render(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_ * getAliasDependency) */ cbm_helm_chart_meta_t meta = c->meta; meta.name = as_name ? as_name : meta.name; + /* .Template.BasePath: the chart's path (full) and templates/, as Helm */ + meta.base_path = full ? cbm_arena_sprintf(&h->a, "%s/templates", full) : NULL; + meta.dir = c->dir; cbm_helm_render(&h->a, t[from].file, src, len, values ? values : c->values, &meta, defs, probes, np); (void)to; @@ -4354,68 +4354,8 @@ static void k8s_helm_emit_uses(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, const k8s } } -/* Do values paths a and b overlap -- one is the other or lies within it - * (a key boundary `.` / `[` after the shorter; "" is the whole values)? */ -static bool k8s_path_overlap(const char *a, const char *b) { - size_t al = strlen(a); - size_t bl = strlen(b); - size_t m = al < bl ? al : bl; - if (strncmp(a, b, m) != 0) { - return false; - } - char c = (al < bl ? b : a)[m]; - return m == 0 || c == '\0' || c == '.' || c == '['; -} - -/* Does values path `path` (of the release's chart) overlap a set_sensitive - * name of the release rendered? */ -static bool k8s_helm_sensitive_path(const k8s_helm_t *h, const char *path) { - if (!h->sensitive || !path) { - return false; - } - yyjson_doc *d = yyjson_read(h->sensitive, strlen(h->sensitive), 0); - bool hit = !d; /* names not read: say it touches */ - size_t idx, max; - yyjson_val *k; - yyjson_arr_foreach(d ? yyjson_doc_get_root(d) : NULL, idx, max, k) { - const char *s = yyjson_get_str(k); - hit = hit || (s && k8s_path_overlap(path, s)); - } - if (d) { - yyjson_doc_free(d); - } - return hit; -} - -/* Does an image reading values keys `keys` (a JSON array, of its chart at - * key path `prefix` within the release's; global.* are the release's own) - * read a set_sensitive name of the release rendered? */ -static bool k8s_helm_sensitive_keys(k8s_helm_t *h, const char *keys, const char *prefix) { - if (!h->sensitive || !keys) { - return false; - } - yyjson_doc *d = yyjson_read(keys, strlen(keys), 0); - bool hit = !d; - size_t idx, max; - yyjson_val *k; - yyjson_arr_foreach(d ? yyjson_doc_get_root(d) : NULL, idx, max, k) { - const char *key = yyjson_get_str(k); - bool global = key && strncmp(key, "global.", strlen("global.")) == 0; - const char *full = !key ? NULL - : global || !prefix || !prefix[0] - ? key - : cbm_arena_sprintf(&h->a, "%s.%s", prefix, key); - hit = hit || !full || k8s_helm_sensitive_path(h, full); - } - if (d) { - yyjson_doc_free(d); - } - return hit; -} - /* Chart `from` -DEPLOYS {via, ref: image, image, override_of}-> the images an - * overlay changed: rendered there, absent from the defaults' images. An image - * reading a set_sensitive name (whose value is not read) is not known. */ + * overlay changed: rendered there, absent from the defaults' images. */ static void k8s_helm_emit_overrides(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t from, const char *via, const k8s_himgs_t *dflt, const k8s_himgs_t *got, yyjson_mut_val *overlay, @@ -4429,7 +4369,7 @@ static void k8s_helm_emit_overrides(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int6 strcmp(dflt->v[j].ref, x->ref) == 0; } const char *ovr = k8s_helm_overridden(h, x->keys, overlay, globals, prefix); - if (same || !ovr || from <= 0 || k8s_helm_sensitive_keys(h, x->keys, prefix)) { + if (same || !ovr || from <= 0) { continue; } k8s_himg_t e = {from, x->img, x->ref, NULL, ovr}; @@ -4520,12 +4460,14 @@ static bool k8s_helm_disabled(k8s_helm_t *h, yyjson_mut_val *dep, yyjson_mut_val * them, each subchart's defaults with its parent's `:` values and * `global:` coalesced over them (Helm's coalesce; a subchart always has a * `global`). *path gets dir's key path from top ("" for top itself), *as_name - * the name dir's chart is included under (its alias; NULL for top). NULL: a - * dependency condition disables it, or a values file does not parse. */ + * the name dir's chart is included under (its alias; NULL for top), *full its + * chart path as Helm's ChartFullPath (top's name, then `/charts/` + * per chart down; NULL: not known). NULL: a dependency condition disables it, + * or a values file does not parse. */ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, k8s_dep_link_t *l, const char *dir, const char *top, yyjson_mut_val *top_vals, const char **path, - const char **as_name) { + const char **as_name, const char **full) { const char **anc = NULL; int na = k8s_helm_ancestors(h, b, dir, &anc); int ti = -SKIP_ONE; /* top's index among the ancestors; na: dir itself */ @@ -4538,6 +4480,7 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, } *path = ""; *as_name = NULL; + *full = NULL; if (ti < 0) { return NULL; } @@ -4545,6 +4488,7 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, if (!cur_chart || !cur_chart->values) { return NULL; } + *full = cur_chart->meta.name; yyjson_mut_val *cur = top_vals ? cbm_helm_values_coalesce(h->doc, cur_chart->values, top_vals) : cur_chart->values; for (int i = (ti == na ? na : ti) - SKIP_ONE; i >= -SKIP_ONE && ti != na; i--) { @@ -4564,6 +4508,7 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, *path = (*path)[0] ? cbm_arena_sprintf(&h->a, "%s.%s", *path, alias) : cbm_arena_strdup(&h->a, alias); *as_name = alias; + *full = *full ? cbm_arena_sprintf(&h->a, "%s/charts/%s", *full, alias) : NULL; yyjson_mut_val *sub = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, alias) : NULL; yyjson_mut_val *next = cbm_helm_values_coalesce(h->doc, child->values, sub); yyjson_mut_val *pg = yyjson_mut_is_obj(cur) ? yyjson_mut_obj_get(cur, "global") : NULL; @@ -4583,10 +4528,10 @@ static yyjson_mut_val *k8s_helm_context(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, /* Render the templated containers [from, to) of one template with `vals` * and take each container's images. */ static void k8s_helm_images_of(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, k8s_bundles_t *b, - const k8s_hchart_t *c, const char *as_name, int from, int to, - yyjson_mut_val *vals, const char *src, size_t len, + const k8s_hchart_t *c, const char *as_name, const char *full, + int from, int to, yyjson_mut_val *vals, const char *src, size_t len, cbm_helm_probe_t *probes, int np, k8s_himgs_t *out) { - k8s_helm_render(ctx, h, b, c, as_name, h->t, from, to, vals, src, len, probes, np); + k8s_helm_render(ctx, h, b, c, as_name, full, h->t, from, to, vals, src, len, probes, np); for (int i = from; i < to; i++) { k8s_helm_take(ctx, probes, np, h->t[i].start_line, h->t[i].end_line, h->t[i].node, h->t[i].container, out); @@ -4614,18 +4559,19 @@ static void k8s_helm_link_defaults(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 : NULL; const char *path = NULL; const char *as_name = NULL; + const char *full = NULL; yyjson_mut_val *vals = NULL; if (c) { k8s_chart_family(b, c->dir, fam, sizeof(fam)); - vals = k8s_helm_context(ctx, h, b, &l, c->dir, fam, NULL, &path, &as_name); + vals = k8s_helm_context(ctx, h, b, &l, c->dir, fam, NULL, &path, &as_name, &full); } int len = 0; char *src = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; cbm_helm_probe_t *probes = NULL; int np = src ? k8s_helm_probes(h, h->t, from, to, src, (size_t)len, &probes) : 0; if (np > 0) { - k8s_helm_images_of(ctx, h, b, c, as_name, from, to, vals, src, (size_t)len, probes, np, - &uses); + k8s_helm_images_of(ctx, h, b, c, as_name, full, from, to, vals, src, (size_t)len, + probes, np, &uses); } cbm_free(CBM_MEM_CLASS_OTHER, src); from = to; @@ -4680,16 +4626,17 @@ static void k8s_helm_release_local(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 k8s_hchart_t *c = in ? k8s_helm_chart(ctx, h, l, dir) : NULL; const char *path = NULL; const char *as_name = NULL; + const char *full = NULL; yyjson_mut_val *vals = - c ? k8s_helm_context(ctx, h, b, l, dir, top, rv, &path, &as_name) : NULL; + c ? k8s_helm_context(ctx, h, b, l, dir, top, rv, &path, &as_name, &full) : NULL; int len = 0; char *srcs = vals ? k8s_repo_read(ctx, h->t[from].file, &len) : NULL; cbm_helm_probe_t *probes = NULL; int np = srcs ? k8s_helm_probes(h, h->t, from, to, srcs, (size_t)len, &probes) : 0; if (np > 0) { k8s_himgs_t got = {0}; - k8s_helm_images_of(ctx, h, b, c, as_name, from, to, vals, srcs, (size_t)len, probes, np, - &got); + k8s_helm_images_of(ctx, h, b, c, as_name, full, from, to, vals, srcs, (size_t)len, + probes, np, &got); yyjson_mut_val *overlay = path && path[0] ? k8s_helm_dig(rv, path) : rv; yyjson_mut_val *globals = yyjson_mut_obj_get(rv, "global"); k8s_helm_emit_overrides(ctx, h, src, via, &h->dflt, &got, overlay, globals, @@ -4748,9 +4695,7 @@ static void k8s_helm_convention(cbm_pipeline_ctx_t *ctx, k8s_helm_t *h, int64_t : "", dig[0] ? dig : tag) : NULL; - /* a set_sensitive name within the mapping: its value is not read */ - bool sensitive = k8s_helm_sensitive_path(h, path ? path : ""); - int64_t img = ref && !sensitive && k8s_helm_has_host(ref) && cbm_image_ref_valid(ref) + int64_t img = ref && k8s_helm_has_host(ref) && cbm_image_ref_valid(ref) ? cbm_pipeline_image_node(ctx, ref) : 0; int before = emitted->n; @@ -4987,12 +4932,9 @@ static void k8s_helm_link_releases(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, k8 if (jd) { yyjson_doc_free(jd); } - h->sensitive = r->sensitive; k8s_helm_release(ctx, b, h, &l, r->src, "terraform", r->chart, rv); - h->sensitive = NULL; cbm_free(CBM_MEM_CLASS_OTHER, r->values); } - cbm_free(CBM_MEM_CLASS_OTHER, r->sensitive); } cbm_free(CBM_MEM_CLASS_OTHER, ctx->tf_releases); ctx->tf_releases = NULL; diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c index a320f6071..e9abd9c38 100644 --- a/src/pipeline/pass_terraform.c +++ b/src/pipeline/pass_terraform.c @@ -645,8 +645,8 @@ static yyjson_mut_val *tf_set_block(const tf_t *t, yyjson_mut_doc *doc, TSNode b return entry; } -/* Is block `a` a `dynamic "set"` / `"set_list"` / `"set_sensitive"` -- - * entries (and names) made by a for_each, never literal? */ +/* Is block `a` a `dynamic "set"` / `"set_list"` / `"set_sensitive"` / + * `"postrender"` -- made by a for_each, never literal? */ static bool tf_dynamic_set(const tf_t *t, TSNode a) { TSNode kw = ts_node_named_child(a, 0); TSNode label = ts_node_named_child(a, SKIP_ONE); @@ -663,7 +663,8 @@ static bool tf_dynamic_set(const tf_t *t, TSNode a) { return (ll == strlen("\"set\"") && strncmp(t->src + lb, "\"set\"", ll) == 0) || (ll == strlen("\"set_list\"") && strncmp(t->src + lb, "\"set_list\"", ll) == 0) || (ll == strlen("\"set_sensitive\"") && - strncmp(t->src + lb, "\"set_sensitive\"", ll) == 0); + strncmp(t->src + lb, "\"set_sensitive\"", ll) == 0) || + (ll == strlen("\"postrender\"") && strncmp(t->src + lb, "\"postrender\"", ll) == 0); } /* Is node n an identifier spelling `word`? */ @@ -676,57 +677,15 @@ static bool tf_word(const tf_t *t, TSNode n, const char *word) { return l == strlen(word) && strncmp(t->src + b, word, l) == 0; } -/* The expression of attribute `name` of a block's body or of an HCL object - * (`{ name = ..., value = ... }`); a null node when there is none. */ -static TSNode tf_entry_attr(const tf_t *t, TSNode entry, const char *name) { - TSNode e = tf_unwrap(entry); - for (uint32_t j = 0; j < ts_node_named_child_count(e); j++) { - TSNode c = ts_node_named_child(e, j); - if (tf_type(e, "block") && tf_type(c, "body")) { - for (uint32_t k = 0; k < ts_node_named_child_count(c); k++) { - TSNode at = ts_node_named_child(c, k); - if (tf_type(at, "attribute") && ts_node_named_child_count(at) >= PAIR_LEN && - tf_word(t, ts_node_named_child(at, 0), name)) { - return ts_node_named_child(at, SKIP_ONE); - } - } - } else if (tf_type(e, "object") && tf_type(c, "object_elem")) { - TSNode kn = tf_unwrap(ts_node_child_by_field_name(c, "key", (uint32_t)strlen("key"))); - TSNode id = tf_type(kn, "variable_expr") ? ts_node_named_child(kn, 0) : (TSNode){0}; - if (tf_word(t, id, name)) { - return ts_node_child_by_field_name(c, "val", (uint32_t)strlen("val")); - } - } - } - return (TSNode){0}; -} - -/* The literal name of set_sensitive entry `entry` (a block, or an object of - * the provider v3 list) appended to names -- the name only, the value is - * never read; false when the name is no literal. */ -static bool tf_sensitive_name(const tf_t *t, yyjson_mut_doc *doc, yyjson_mut_val *names, - TSNode entry) { - yyjson_mut_val *nm = tf_json(t, doc, tf_entry_attr(t, entry, "name")); - if (!yyjson_mut_is_str(nm)) { - return false; - } - yyjson_mut_arr_append(names, nm); - return true; -} - /* The values a helm_release body gives its chart -- `values` items * coalesced in order, then each `set` -- as JSON (cbm_alloc); NULL when a - * part is no literal, is made by a `dynamic` block, or a postrender may - * rewrite what it deploys. set_sensitive values are never read: their names - * go to *sensitive (a JSON array, cbm_alloc; NULL: none), and an image that - * reads one is not known (pass_k8s.c). */ -static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles, - char **sensitive) { - *sensitive = NULL; + * part is no literal, is made by a `dynamic` block, is sensitive (its value + * is never read), or a postrender -- a block, an attribute or a dynamic one + * -- may rewrite what it deploys. */ +static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, int *nfiles) { yyjson_mut_doc *doc = yyjson_mut_doc_new(cbm_json_writer_allocator()); yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; - yyjson_mut_val *names = doc ? yyjson_mut_arr(doc) : NULL; - bool ok = root != NULL && names != NULL; + bool ok = root != NULL; for (int pass = 0; ok && pass < PAIR_LEN; pass++) { for (uint32_t i = 0; ok && i < ts_node_named_child_count(body); i++) { TSNode a = ts_node_named_child(body, i); @@ -735,31 +694,14 @@ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, size_t kl = ts_node_is_null(key) ? 0 : ts_node_end_byte(key) - kb; bool is_set = kl == strlen("set") && strncmp(t->src + kb, "set", kl) == 0; bool is_values = kl == strlen("values") && strncmp(t->src + kb, "values", kl) == 0; - /* set_list and dynamic set values are not read: the release's - * values would be partial; a postrender may rewrite its images */ - if ((kl == strlen("set_list") && strncmp(t->src + kb, "set_list", kl) == 0) || - tf_dynamic_set(t, a) || (tf_type(a, "block") && tf_word(t, key, "postrender"))) { + /* set_list, set_sensitive and dynamic set values are not read: + * the release's values would be partial; a postrender may + * rewrite its images */ + if (tf_word(t, key, "set_list") || tf_word(t, key, "set_sensitive") || + tf_word(t, key, "postrender") || tf_dynamic_set(t, a)) { ok = false; break; } - bool is_sensitive = - kl == strlen("set_sensitive") && strncmp(t->src + kb, "set_sensitive", kl) == 0; - if (pass == 0 && is_sensitive && tf_type(a, "block")) { - ok = tf_sensitive_name(t, doc, names, a); - continue; - } - if (pass == 0 && is_sensitive && tf_type(a, "attribute")) { - /* provider v3: set_sensitive = [{ name, value }, ...] */ - TSNode list = tf_unwrap(ts_node_named_child(a, SKIP_ONE)); - ok = tf_type(list, "tuple"); - for (uint32_t j = 0; ok && j < ts_node_named_child_count(list); j++) { - TSNode it = ts_node_named_child(list, j); - if (!tf_type(it, "tuple_start") && !tf_type(it, "tuple_end")) { - ok = tf_sensitive_name(t, doc, names, it); - } - } - continue; - } if (pass == 0 && is_values && tf_type(a, "attribute")) { TSNode list = tf_unwrap(ts_node_named_child(a, SKIP_ONE)); ok = tf_type(list, "tuple"); @@ -791,15 +733,6 @@ static char *tf_release_values(const tf_t *t, TSNode body, const char ***files, if (ok) { size_t len = 0; out = yyjson_mut_val_write_opts(root, 0, cbm_json_writer_allocator(), &len, NULL); - if (out && yyjson_mut_arr_size(names) > 0) { - *sensitive = - yyjson_mut_val_write_opts(names, 0, cbm_json_writer_allocator(), &len, NULL); - if (!*sensitive) { - /* the names not kept: no image of the release could be told apart */ - cbm_free(CBM_MEM_CLASS_OTHER, out); - out = NULL; - } - } } if (doc) { yyjson_mut_doc_free(doc); @@ -821,9 +754,8 @@ static void tf_release_push(tf_t *t, int64_t src, int64_t chart, TSNode body) { } const char **files = NULL; int nfiles = 0; - char *sensitive = NULL; - char *values = tf_release_values(t, body, &files, &nfiles, &sensitive); - ctx->tf_releases[ctx->tf_release_n++] = (cbm_tf_release_t){src, chart, values, sensitive}; + char *values = tf_release_values(t, body, &files, &nfiles); + ctx->tf_releases[ctx->tf_release_n++] = (cbm_tf_release_t){src, chart, values}; /* the values files it reads, like a kustomization's valuesFile: an edit of * one alone renders the release again (pipeline_incremental.c) */ for (int i = 0; i < nfiles; i++) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 402474c20..553aa19d9 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -385,10 +385,9 @@ typedef struct { * when a part is not a literal (an interpolation, a variable). pass_k8s.c * renders the chart's images with them. */ typedef struct cbm_tf_release { - int64_t src; /* the resource.helm_release node */ - int64_t chart; /* the Chart node */ - char *values; /* owned (cbm_free); NULL: unknown */ - char *sensitive; /* owned: JSON array of its set_sensitive names (never values); NULL: none */ + int64_t src; /* the resource.helm_release node */ + int64_t chart; /* the Chart node */ + char *values; /* owned (cbm_free); NULL: unknown */ } cbm_tf_release_t; /* Shared context passed to each pass function. diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 7b89c755a..0c8cf008e 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -4871,9 +4871,10 @@ TEST(probe_helm_release_terraform_dynamic_set) { PASS(); } -/* A set_sensitive whose value is never read: a release image reading its - * name (image.tag) is not known; one reading none of them still is. A - * postrender may rewrite what a release deploys: no image of its own. */ +/* A set_sensitive's value is never read: any set_sensitive leaves the + * release without an image of its own, whatever its name -- image.tag or + * db.password (which kept its edge in round 3). A postrender may rewrite + * what a release deploys: no image of its own either. */ TEST(probe_helm_release_terraform_sensitive_and_postrender) { static const GpfFile files[] = { GPF_APP_CHART, @@ -4889,15 +4890,105 @@ TEST(probe_helm_release_terraform_sensitive_and_postrender) { "resource \"helm_release\" \"post\" {\n name = \"post\"\n" " chart = \"../charts/app\"\n set {\n name = \"image.tag\"\n" " value = \"3.3\"\n }\n postrender {\n binary_path = \"./kustomize.sh\"\n }\n}\n"}}; + if (!gpf_expect_image_edges(files, 4, GPF_APP_DEFAULTS)) { + FAIL("any set_sensitive and a postrender make release images unknown"); + } + PASS(); +} + +/* A set_sensitive whose name an image reads on another line ($tag, before + * the image line): the release still has no image of its own -- never one + * with the chart's default tag. */ +TEST(probe_helm_release_terraform_sensitive_read_earlier) { + static const GpfFile files[] = { + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 1.0.0\nappVersion: \"1.0\"\n"}, + {"charts/web/values.yaml", "image:\n repository: ghcr.io/acme/web\n"}, + {"charts/web/templates/deploy.yaml", + "{{- $tag := .Values.image.tag | default .Chart.AppVersion }}\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " spec:\n containers:\n - name: web\n" + " image: \"{{ .Values.image.repository }}:{{ $tag }}\"\n"}, + {"tf/main.tf", + "resource \"helm_release\" \"web\" {\n name = \"web\"\n" + " chart = \"../charts/web\"\n set {\n name = \"image.repository\"\n" + " value = \"quay.io/acme/web\"\n }\n set_sensitive {\n name = \"image.tag\"\n" + " value = var.tag\n }\n}\n"}}; + static const char want[] = + "Chart:web -> Image:ghcr.io/acme/web DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.repository\"]\n" + "Resource:Deployment/web -> Image:ghcr.io/acme/web USAGE image=ghcr.io/acme/web:1.0 " + "ref=image default_of=[\"image.repository\"]"; + if (!gpf_expect_image_edges(files, 4, want)) { + FAIL("a set_sensitive read on another line makes the release images unknown"); + } + PASS(); +} + +/* A postrender as a dynamic block or as an attribute (provider v3) may + * rewrite what a release deploys, as a block does: no image of its own. */ +TEST(probe_helm_release_terraform_postrender_forms) { + static const GpfFile files[] = { + GPF_APP_CHART, + {"tf/main.tf", + "resource \"helm_release\" \"dyn\" {\n name = \"dyn\"\n" + " chart = \"../charts/app\"\n set {\n name = \"image.tag\"\n" + " value = \"3.3\"\n }\n dynamic \"postrender\" {\n for_each = var.post\n" + " content {\n binary_path = postrender.value\n }\n }\n}\n" + "resource \"helm_release\" \"attr\" {\n name = \"attr\"\n" + " chart = \"../charts/app\"\n set = [{ name = \"image.tag\", value = \"4.4\" }]\n" + " postrender = { binary_path = \"./kustomize.sh\" }\n}\n"}}; + if (!gpf_expect_image_edges(files, 4, GPF_APP_DEFAULTS)) { + FAIL("a dynamic or attribute postrender makes release images unknown"); + } + PASS(); +} + +/* A subchart included under an alias (redis as `cache`) has .Template.BasePath + * app/charts/cache/templates, as Helm sets it (an image here shows it). Its + * checksum of cm.yaml names its own file, which changes no map -- not app's + * cm.yaml, which sets a value: its image stays known, as does app's (which + * checksums its Secret). */ +TEST(probe_helm_checksum_include_by_base_path) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\ndependencies:\n" + " - name: redis\n version: 1.0.0\n alias: cache\n"}, + {"charts/app/values.yaml", "image:\n repository: ghcr.io/acme/app\n tag: \"1.0\"\n"}, + {"charts/app/templates/cm.yaml", "{{- $_ := set .Values.image \"tag\" \"2.0\" }}\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: app\n"}, + {"charts/app/templates/secret.yaml", "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n"}, + {"charts/app/templates/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: app\nspec:\n template:\n" + " metadata:\n annotations:\n checksum/secret: " + "{{ include (print $.Template.BasePath \"/secret.yaml\") . | sha256sum }}\n" + " spec:\n containers:\n - name: app\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"}, + {"charts/app/charts/redis/Chart.yaml", "apiVersion: v2\nname: redis\nversion: 1.0.0\n"}, + {"charts/app/charts/redis/values.yaml", + "image:\n repository: ghcr.io/acme/redis\n tag: \"7.0\"\n"}, + {"charts/app/charts/redis/templates/cm.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: redis\n"}, + {"charts/app/charts/redis/templates/sts.yaml", + "apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n name: redis\nspec:\n template:\n" + " metadata:\n annotations:\n checksum/config: " + "{{ include (print $.Template.BasePath \"/cm.yaml\") . | sha256sum }}\n" + " spec:\n containers:\n - name: redis\n" + " image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n" + " - name: path\n image: \"ghcr.io/{{ .Template.BasePath }}:1\"\n"}}; static const char want[] = "Chart:app -> Image:ghcr.io/acme/app DEPLOYS image=- via=chart ref=image " "default_of=[\"image.repository\",\"image.tag\"]\n" - "Class:resource.helm_release.pw -> Image:quay.io/acme/pw DEPLOYS " - "image=quay.io/acme/pw:1.0 via=terraform ref=image override_of=[\"image.repository\"]\n" + "Chart:redis -> Image:ghcr.io/acme/redis DEPLOYS image=- via=chart ref=image " + "default_of=[\"image.repository\",\"image.tag\"]\n" + "Chart:redis -> Image:ghcr.io/app/charts/cache/templates DEPLOYS image=- via=chart " + "ref=image\n" "Resource:Deployment/app -> Image:ghcr.io/acme/app USAGE image=ghcr.io/acme/app:1.0 " - "ref=image default_of=[\"image.repository\",\"image.tag\"]"; - if (!gpf_expect_image_edges(files, 4, want)) { - FAIL("set_sensitive names and postrender make release images unknown"); + "ref=image default_of=[\"image.repository\",\"image.tag\"]\n" + "Resource:StatefulSet/redis -> Image:ghcr.io/acme/redis USAGE " + "image=ghcr.io/acme/redis:7.0 ref=image default_of=[\"image.repository\",\"image.tag\"]\n" + "Resource:StatefulSet/redis -> Image:ghcr.io/app/charts/cache/templates USAGE " + "image=ghcr.io/app/charts/cache/templates:1 ref=image"; + if (!gpf_expect_image_edges(files, 9, want)) { + FAIL("a checksum include by .Template.BasePath names the chart's own file"); } PASS(); } @@ -7013,6 +7104,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_helm_release_kustomize_unread_sources); RUN_TEST(probe_helm_release_terraform_dynamic_set); RUN_TEST(probe_helm_release_terraform_sensitive_and_postrender); + RUN_TEST(probe_helm_release_terraform_sensitive_read_earlier); + RUN_TEST(probe_helm_release_terraform_postrender_forms); + RUN_TEST(probe_helm_checksum_include_by_base_path); RUN_TEST(probe_helm_release_kustomize_transformers); RUN_TEST(probe_helm_empty_values_file_renders); RUN_TEST(probe_helm_aliased_subchart_chart_name); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index dc03c8cf7..aef677834 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4338,14 +4338,15 @@ TEST(helm_values_follow_helm_semantics) { /* Render template tpl with values src (one probe per `image:` line given) * and return the capture of probe 0 ("" for UNKNOWN, "" when * nothing was captured) -- a copy, freed by the caller. */ -static char *helm_capture(const char *vals_src, const char *tpl, const cbm_helm_defines_t *defs, - int line) { +static char *helm_capture_chart(const char *vals_src, const char *tpl, + const cbm_helm_defines_t *defs, const cbm_helm_chart_meta_t *meta, + int line) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *v = doc ? cbm_helm_values_parse(doc, vals_src, strlen(vals_src)) : NULL; CBMArena a; cbm_arena_init_lazy(&a, CBM_SZ_4K); cbm_helm_probe_t probe = {.line = line}; - cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, NULL, defs, &probe, 1); + cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, meta, defs, &probe, 1); char *out = strdup(probe.n == 0 ? "" : probe.image[0] ? probe.image[0] : ""); cbm_arena_destroy(&a); if (doc) { @@ -4354,6 +4355,12 @@ static char *helm_capture(const char *vals_src, const char *tpl, const cbm_helm_ return out; } +/* helm_capture_chart of a template of no chart. */ +static char *helm_capture(const char *vals_src, const char *tpl, const cbm_helm_defines_t *defs, + int line) { + return helm_capture_chart(vals_src, tpl, defs, NULL, line); +} + /* A render doing far more work than any chart -- here a range over an int * of 20,000,000 -- stops at its step budget (helm_template.c): every capture * of the template is UNKNOWN, also the one made before the loop, never a @@ -4540,10 +4547,12 @@ TEST(helm_unknown_break_leaves_the_loop) { PASS(); } -/* A change made to a new value -- fromYaml / deepCopy results, a part of one, - * an empty dict -- reaches nothing another part holds: the images after it - * stay known (the opentelemetry-ebpf-instrumentation configData pattern). */ -TEST(helm_change_of_a_new_value_spoils_nothing) { +/* A change of maps the render does not run exactly leaves every image after + * it UNKNOWN -- also one of a new value (fromYaml / deepCopy results, an empty + * dict): no exemption (the coarse rule, decided in review round 3). Rounds 1-3 + * kept this image (the opentelemetry-ebpf-instrumentation configData + * pattern); it is part of the rule's measured cost. */ +TEST(helm_change_of_a_new_value_spoils_after) { static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\ncfg: {a: 1}\n"; static const char tpl[] = "{{- $config := dict -}}\n" @@ -4554,7 +4563,7 @@ TEST(helm_change_of_a_new_value_spoils_nothing) { "{{- $_ := unset (deepCopy .Values.cfg) \"a\" }}\n" "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; char *got = helm_capture(vals, tpl, NULL, 6); - ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(got, ""); free(got); PASS(); } @@ -4661,7 +4670,9 @@ TEST(helm_unknown_break_spoils_the_rest_of_the_loop) { /* A variable once made from `dict` and then (in a body an UNKNOWN condition * skips) redefined or given .Values.image is no new map: a set of it there - * may change .Values. Without the redefinition the set is of the new map. */ + * may change .Values. Without the redefinition the set is of the new map -- + * still a change in a skipped body, which (the coarse rule) leaves every + * image after it UNKNOWN; rounds 2-3 kept that one's image. */ TEST(helm_skipped_set_after_redefinition) { static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; static const char redefined[] = @@ -4682,7 +4693,7 @@ TEST(helm_skipped_set_after_redefinition) { ASSERT_STR_EQ(got, ""); free(got); got = helm_capture(vals, kept, NULL, 2); - ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(got, ""); free(got); PASS(); } @@ -4744,6 +4755,301 @@ TEST(helm_include_with_poisoned_argument) { PASS(); } +/* The keys a capture read -- 5,000 values paths -- become its mark in one + * pass: memory linear in them, never a copy of the mark so far per key. */ +TEST(helm_capture_keys_cost_linear_memory) { + enum { ITEMS = 5000 }; + size_t cap = (size_t)ITEMS * 8 + CBM_SZ_64; + char *vals = malloc(cap); + ASSERT_NOT_NULL(vals); + size_t o = (size_t)snprintf(vals, cap, "big: ["); + for (int i = 0; i < ITEMS; i++) { + o += (size_t)snprintf(vals + o, cap - o, "%s{x: a}", i ? "," : ""); + } + snprintf(vals + o, cap - o, "]\n"); + static const char tpl[] = "image: \"r/{{ range .Values.big }}{{ .x }}{{ end }}\"\n"; + cbm_mem_class_flush_thread(); + size_t base = cbm_mem_class_live_bytes(CBM_MEM_CLASS_ARENA); + cbm_mem_class_reset_peaks(); + char *got = helm_capture(vals, tpl, NULL, 1); + cbm_mem_class_flush_thread(); + size_t peak = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_ARENA); + free(vals); + ASSERT_NOT_NULL(got); + bool image = strncmp(got, "r/aaaa", strlen("r/aaaa")) == 0; + free(got); + ASSERT_TRUE(image); + /* about 14 bytes a key: ~70 KB of keys; a copy per key is ~175 MB */ + ASSERT_TRUE(peak - base < (size_t)CBM_SZ_32 * CBM_SZ_1K * CBM_SZ_1K); + PASS(); +} + +/* A render's work -- here 20 copies of a 20,000-item list -- stays in an + * arena of its own: the caller's arena grows by the render's results only, + * so renders one after another never keep each other's work. */ +TEST(helm_render_keeps_only_its_results) { + enum { ITEMS = 20000 }; + size_t cap = (size_t)ITEMS * 2 + CBM_SZ_64; + char *vals = malloc(cap); + ASSERT_NOT_NULL(vals); + size_t o = (size_t)snprintf(vals, cap, "big: ["); + for (int i = 0; i < ITEMS; i++) { + o += (size_t)snprintf(vals + o, cap - o, "%s1", i ? "," : ""); + } + snprintf(vals + o, cap - o, "]\n"); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *v = doc ? cbm_helm_values_parse(doc, vals, strlen(vals)) : NULL; + free(vals); + static const char tpl[] = "image: \"ghcr.io/acme/app:1\"\n" + "{{- range 20 }}{{ $x := first $.Values.big }}{{ end }}\n"; + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_4K); + cbm_helm_probe_t probe = {.line = 1}; + size_t before = cbm_arena_total(&a); + bool ok = cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, NULL, NULL, &probe, 1); + size_t kept = cbm_arena_total(&a) - before; + bool image = ok && probe.n == 1 && probe.image[0] && + strcmp(probe.image[0], "ghcr.io/acme/app:1") == 0; + cbm_arena_destroy(&a); + if (doc) { + yyjson_mut_doc_free(doc); + } + ASSERT_NOT_NULL(v); + ASSERT_TRUE(image); + ASSERT_TRUE(kept < (size_t)CBM_SZ_1K * CBM_SZ_1K); /* the work: ~30 MB */ + PASS(); +} + +static char g_helm_budget_log[CBM_SZ_4K]; + +static void helm_budget_log_sink(const char *line) { + if (!line || !strstr(line, "helm.render.budget")) { + return; + } + size_t used = strlen(g_helm_budget_log); + snprintf(g_helm_budget_log + used, sizeof(g_helm_budget_log) - used, "%s\n", line); +} + +/* A render its output budget stopped -- an indent that writes past 64 MiB, + * or one too wide to try -- says so: reason=output. */ +TEST(helm_render_budget_logs_its_reason) { + g_helm_budget_log[0] = '\0'; + CBMLogLevel prev = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_DEBUG); + cbm_log_set_sink(helm_budget_log_sink); + static const char big[] = "image: \"ghcr.io/acme/app{{ indent 70000000 \"\" | trim }}:1\"\n"; + char *got = helm_capture("{}\n", big, NULL, 1); + free(got); + static const char huge[] = + "image: \"ghcr.io/acme/app{{ indent 9000000000000000000 \"\" | trim }}:1\"\n"; + got = helm_capture("{}\n", huge, NULL, 1); + free(got); + cbm_log_set_sink(NULL); + cbm_log_set_level(prev); + const char *first = strstr(g_helm_budget_log, "reason=output"); + ASSERT_NOT_NULL(first); + ASSERT_NOT_NULL(strstr(first + SKIP_ONE, "reason=output")); + ASSERT_NULL(strstr(g_helm_budget_log, "reason=memory")); + PASS(); +} + +/* A merge into .Values may change any map of the values, also one a handle + * ($img) was taken of before: every image after it is UNKNOWN. */ +TEST(helm_merge_into_values_root) { + static const char vals[] = "image: {repository: ghcr.io/acme/app}\n"; + static const char tpl[] = + "{{- $img := .Values.image }}\n" + "{{- $_ := merge .Values (dict \"image\" (dict \"tag\" \"2.0\")) }}\n" + "image: \"{{ $img.repository }}:{{ $img.tag | default \"1.0\" }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A set in a body an UNKNOWN condition skips, whatever its target -- here + * the outer $vals (.Values.image), a nested range's own $vals aside -- + * leaves every image after it UNKNOWN. */ +TEST(helm_skipped_set_beside_a_nested_scope) { + static const char vals[] = + "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\njobs: [a]\n"; + static const char tpl[] = "{{- $vals := .Values.image }}\n" + "{{- if .Capabilities.APIVersions.Has \"batch/v1\" }}\n" + "{{- range .Values.jobs }}{{- $vals := dict }}{{- end }}\n" + "{{- $_ := set $vals \"tag\" \"2.0\" }}\n" + "{{- end }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 6); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* An UNKNOWN break in a range over maps: whether the later elements ($c of + * web) were changed is not known -- not only the current one's. */ +TEST(helm_unknown_break_spoils_later_elements) { + static const char vals[] = "components:\n" + " api: {repository: ghcr.io/acme/api, tag: \"1.0\"}\n" + " web: {repository: ghcr.io/acme/web, tag: \"1.0\"}\n"; + static const char tpl[] = + "{{- range $n, $c := .Values.components }}\n" + "{{- if semverCompare \">=1.29-0\" $.Capabilities.KubeVersion.Version }}{{ break }}" + "{{ end }}\n" + "{{- $_ := set $c \"tag\" \"2.0\" }}\n" + "{{- end }}\n" + "image: \"{{ .Values.components.web.repository }}:{{ .Values.components.web.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 5); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A skipped range whose element variable ($c, each component) is set is no + * change of an outer $c (a new dict): every image after it is UNKNOWN. */ +TEST(helm_skipped_range_sets_its_element) { + static const char vals[] = "components:\n" + " web: {repository: ghcr.io/acme/web, tag: \"1.0\"}\n"; + static const char tpl[] = + "{{- $c := dict }}\n" + "{{- if .Capabilities.APIVersions.Has \"x/v1\" }}{{- range $_, $c := .Values.components }}" + "{{- $_ := set $c \"tag\" \"2.0\" }}{{- end }}{{- end }}\n" + "image: \"{{ .Values.components.web.repository }}:{{ .Values.components.web.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A deep merge into a map that holds a values map by reference ($d.img is + * .Values.image, put by a set) changes that values map: the image after it + * is UNKNOWN. */ +TEST(helm_deep_merge_reaches_a_held_values_map) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + static const char tpl[] = + "{{- $d := dict }}{{- $_ := set $d \"img\" .Values.image }}\n" + "{{- $_ := mergeOverwrite $d (dict \"img\" (dict \"tag\" \"2.0\")) }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n"; + char *got = helm_capture(vals, tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* Template files of chart app (charts/app) and of its subchart redis, which + * app's dependencies alias `cache` (charts/app/charts/redis): a Secret and a + * ConfigMap with no change of maps, an app ConfigMap that sets a value, and + * a file that includes itself. */ +static cbm_helm_defines_t *helm_chart_files(void) { + static const char secret[] = "apiVersion: v1\nkind: Secret\nmetadata:\n" + " name: {{ include \"app.name\" . }}\n"; + static const char helpers[] = "{{- define \"app.name\" -}}{{ .Chart.Name }}{{- end -}}\n"; + static const char app_cm[] = "{{- $_ := set .Values.image \"tag\" \"2.0\" }}\n" + "apiVersion: v1\nkind: ConfigMap\n"; + static const char sub_cm[] = "apiVersion: v1\nkind: ConfigMap\nmetadata:\n" + " name: {{ .Chart.Name }}\n"; + static const char loop[] = "{{ include \"app/templates/loop.yaml\" . }}\n"; + cbm_helm_defines_t *d = cbm_helm_defines_new(); + if (d) { + cbm_helm_defines_add(d, "charts/app/templates/_helpers.tpl", helpers, strlen(helpers)); + cbm_helm_defines_add(d, "charts/app/templates/secret.yaml", secret, strlen(secret)); + cbm_helm_defines_add(d, "charts/app/templates/cm.yaml", app_cm, strlen(app_cm)); + cbm_helm_defines_add(d, "charts/app/templates/loop.yaml", loop, strlen(loop)); + cbm_helm_defines_add(d, "charts/app/charts/redis/templates/cm.yaml", sub_cm, + strlen(sub_cm)); + } + return d; +} + +/* A template of chart `meta` checksumming its file `file`, then an image. */ +static char *helm_capture_checksum(const cbm_helm_defines_t *d, const cbm_helm_chart_meta_t *meta, + const char *file) { + static const char vals[] = "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n"; + char tpl[CBM_SZ_512]; + snprintf(tpl, sizeof(tpl), + "metadata:\n annotations:\n" + " checksum: {{ include (print $.Template.BasePath \"/%s\") . | sha256sum }}\n" + "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n", + file); + return helm_capture_chart(vals, tpl, d, meta, 4); +} + +static const cbm_helm_chart_meta_t HELM_APP = { + .name = "app", .base_path = "app/templates", .dir = "charts/app"}; + +/* The checksum idiom names one of the chart's own template files by its + * .Template.BasePath: one with no change of maps in it (a Secret) spoils + * nothing -- its output is not rendered, the image after it is known. */ +TEST(helm_checksum_include_keeps_the_image) { + cbm_helm_defines_t *d = helm_chart_files(); + ASSERT_NOT_NULL(d); + char *got = helm_capture_checksum(d, &HELM_APP, "secret.yaml"); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + free(got); + PASS(); +} + +/* An included template file that sets a value (cm.yaml) may change any map: + * every image after it is UNKNOWN -- beside one that does not (secret.yaml). */ +TEST(helm_include_of_a_changing_template_file) { + cbm_helm_defines_t *d = helm_chart_files(); + ASSERT_NOT_NULL(d); + char *clean = helm_capture_checksum(d, &HELM_APP, "secret.yaml"); + char *changing = helm_capture_checksum(d, &HELM_APP, "cm.yaml"); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(clean, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(changing, ""); + free(clean); + free(changing); + PASS(); +} + +/* A subchart's .Template.BasePath -- app/charts/cache/templates, by the alias + * it is included under -- names its own cm.yaml (no change), not app's (a + * set): its image is known; app's own checksum of cm.yaml is not. */ +TEST(helm_subchart_base_path_names_its_own_file) { + cbm_helm_defines_t *d = helm_chart_files(); + ASSERT_NOT_NULL(d); + const cbm_helm_chart_meta_t sub = {.name = "cache", + .base_path = "app/charts/cache/templates", + .dir = "charts/app/charts/redis"}; + char *own = helm_capture_checksum(d, &sub, "cm.yaml"); + char *parent = helm_capture_checksum(d, &HELM_APP, "cm.yaml"); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(own, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(parent, ""); + free(own); + free(parent); + PASS(); +} + +/* An include naming no template file of the chart is not looked into: every + * image after it is UNKNOWN -- beside one naming a file that is there. */ +TEST(helm_include_of_a_missing_template_file) { + cbm_helm_defines_t *d = helm_chart_files(); + ASSERT_NOT_NULL(d); + char *there = helm_capture_checksum(d, &HELM_APP, "secret.yaml"); + char *missing = helm_capture_checksum(d, &HELM_APP, "missing.yaml"); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(there, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(missing, ""); + free(there); + free(missing); + PASS(); +} + +/* A template file that includes itself is never "no change": every image + * after an include of it is UNKNOWN. */ +TEST(helm_template_file_including_itself) { + cbm_helm_defines_t *d = helm_chart_files(); + ASSERT_NOT_NULL(d); + char *got = helm_capture_checksum(d, &HELM_APP, "loop.yaml"); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + /* A compose project's .env feeds its compose file (variables in image, * build): changing it is no closure repair -- the unchanged compose file's * services would keep the old image. It rebuilds; no stale image remains. */ @@ -24481,7 +24787,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(helm_unknown_branch_spoils_what_it_assigns); RUN_TEST(helm_mutator_spoils_the_map_it_changes); RUN_TEST(helm_unknown_break_leaves_the_loop); - RUN_TEST(helm_change_of_a_new_value_spoils_nothing); + RUN_TEST(helm_change_of_a_new_value_spoils_after); RUN_TEST(helm_render_captures_grow_linearly); RUN_TEST(helm_render_memory_budget_ends_unknown); RUN_TEST(helm_skipped_set_of_a_call_result); @@ -24492,6 +24798,19 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(helm_many_changed_maps_keep_reads_right); RUN_TEST(helm_define_memo_under_mutual_recursion); RUN_TEST(helm_include_with_poisoned_argument); + RUN_TEST(helm_capture_keys_cost_linear_memory); + RUN_TEST(helm_render_keeps_only_its_results); + RUN_TEST(helm_render_budget_logs_its_reason); + RUN_TEST(helm_merge_into_values_root); + RUN_TEST(helm_skipped_set_beside_a_nested_scope); + RUN_TEST(helm_unknown_break_spoils_later_elements); + RUN_TEST(helm_skipped_range_sets_its_element); + RUN_TEST(helm_deep_merge_reaches_a_held_values_map); + RUN_TEST(helm_checksum_include_keeps_the_image); + RUN_TEST(helm_include_of_a_changing_template_file); + RUN_TEST(helm_subchart_base_path_names_its_own_file); + RUN_TEST(helm_include_of_a_missing_template_file); + RUN_TEST(helm_template_file_including_itself); RUN_TEST(incremental_compose_dotenv_change_rebuilds); RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges); RUN_TEST(incremental_second_build_declarer_keeps_the_edge); From 74b115e929d458b24ee714c9121d7cf2e8642ea5 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 04:23:06 +0200 Subject: [PATCH 14/19] test(python): a top-level __go(locals()) keeps the package namespace sqlalchemy's sql/__init__.py ends with __go(locals()). The callee is a same-module function that never touches its parameter or the module's globals; only the submodules its body imports become package attributes. The extraction test pins the exact shape (the namespace survives, the imported submodules are shadowed at the call) and its controls: the body uses the parameter, global (also nested), globals(), sys.modules[__name__], the own package name, a same-module helper; an imported, rebound or not yet defined callee; an argument with an effect; a module-level locals. The pipeline fixture adds the same shape end to end, with a re-export the body's submodule import replaces and a __go that writes its parameter. Signed-off-by: Martin Vogel --- tests/test_extraction.c | 111 ++++++++++++++++++++++++++++++++++++++++ tests/test_pipeline.c | 30 ++++++++++- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 3391eeb8a..ebfa48f8c 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -12408,6 +12408,116 @@ TEST(python_namespace_facts_string_statements_and_future_imports) { PASS(); } +/* A top-level call of a same-module function is a namespace effect, unless the + * callee provably cannot rebind the caller's names: sqlalchemy's + * `sql/__init__.py` ends with `__go(locals())`. Every submodule its body + * imports becomes a package attribute, so those names are shadowed at the + * call; everything else of the namespace survives. */ +static bool py_facts_has_effect(const CBMFileResult *r) { + for (int i = 0; r && i < r->py_namespace.count; i++) { + if (r->py_namespace.items[i].kind == CBM_PY_NS_UNKNOWN && + r->py_namespace.items[i].reason == CBM_PY_NS_REASON_EFFECT) + return true; + } + return false; +} + +static int py_facts_count_kind(const CBMFileResult *r, CBMPyNamespaceFactKind kind) { + int n = 0; + for (int i = 0; r && i < r->py_namespace.count; i++) + n += r->py_namespace.items[i].kind == kind; + return n; +} + +static bool py_facts_valid_complete(const CBMFileResult *r) { + return r && !r->has_error && r->py_namespace.status == CBM_PY_NS_COMPLETE && + cbm_py_namespace_facts_valid(&r->py_namespace); +} + +TEST(python_namespace_facts_init_call_allowlist) { + static const char go[] = "from typing import TYPE_CHECKING, Any\n" + "from .impl import base\n" + "from .elements import cast\n" + "def __go(lcls: Any) -> None:\n" + " from .. import util as _sa_util\n" + " from . import base\n" + " from . import coercions, elements as _el\n" + " if not TYPE_CHECKING:\n" + " base.coercions = _el.coercions = coercions\n" + " from .annotation import _prepare_annotations\n" + " _prepare_annotations(cast, cast)\n" + " _sa_util.preloaded.import_prefix(\"sqlalchemy.sql\")\n" + "\n" + "__go(locals())\n"; + CBMFileResult *r = extract(go, CBM_LANG_PYTHON, "t", "sql/__init__.py"); + bool ok = py_facts_valid_complete(r) && !py_facts_has_effect(r) && r->py_namespace.count == 9; + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + static const char *const shadowed[] = {"base", "coercions", "elements", "annotation"}; + ok = f[2].kind == CBM_PY_NS_IMPORT_NAME && strcmp(f[2].local_name, "base") == 0 && + f[4].kind == CBM_PY_NS_OWN_DEF && strcmp(f[4].local_name, "__go") == 0; + for (int i = 0; ok && i < 4; i++) { + ok = f[5 + i].kind == CBM_PY_NS_SHADOW && f[5 + i].local_name && + strcmp(f[5 + i].local_name, shadowed[i]) == 0; + } + } + if (!ok && r) { + for (int i = 0; i < r->py_namespace.count; i++) { + const CBMPyNamespaceFact *f = &r->py_namespace.items[i]; + printf(" __go fact %d: kind %d reason %d name %s\n", i, (int)f->kind, (int)f->reason, + f->local_name ? f->local_name : "-"); + } + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); /* RED while every top-level call withdraws the namespace */ + + /* A plain module is no package: the body's imports set attributes of its + * parent package, never of the module itself. */ + r = extract("def __go(lcls):\n from . import base\n\n__go(locals())\n", CBM_LANG_PYTHON, + "t", "sql/mod.py"); + ok = py_facts_valid_complete(r) && !py_facts_has_effect(r) && + py_facts_count_kind(r, CBM_PY_NS_SHADOW) == 0; + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + + /* Each of these can rebind a name of the caller's namespace (or is not + * provably a same-module function): the call stays a whole-namespace + * effect. */ + static const char *const effects[][2] = { + {"param", "def __go(lcls):\n lcls['base'] = 1\n\n__go(locals())\n"}, + {"global", "def __go(lcls):\n global base\n base = 1\n\n__go(locals())\n"}, + {"nested global", "def __go(lcls):\n def inner():\n global base\n" + " base = 1\n inner()\n\n__go(locals())\n"}, + {"globals()", "def __go(lcls):\n globals()['base'] = 1\n\n__go(locals())\n"}, + {"sys.modules", "def __go(lcls):\n import sys\n sys.modules[__name__].base = 1\n" + "\n__go(locals())\n"}, + {"imported callee", "from .setup import __go\n__go(locals())\n"}, + {"own package name", "def __go(lcls):\n from .. import sql\n sql.base = 1\n" + "\n__go(locals())\n"}, + {"same-module helper", "def helper():\n pass\n\ndef __go(lcls):\n helper()\n" + "\n__go(locals())\n"}, + {"rebound callee", "def __go(lcls):\n pass\n\n__go = other\n__go(locals())\n"}, + {"call before def", "__go(locals())\n\ndef __go(lcls):\n pass\n"}, + {"argument effect", "def __go(lcls):\n pass\n\n__go(make())\n"}, + {"rebound locals", "def locals():\n return {}\n\ndef __go(lcls):\n pass\n" + "\n__go(locals())\n"}, + }; + for (size_t i = 0; i < sizeof(effects) / sizeof(effects[0]); i++) { + char src[512]; + snprintf(src, sizeof(src), "from .impl import base\n%s", effects[i][1]); + r = extract(src, CBM_LANG_PYTHON, "t", "sql/__init__.py"); + ok = py_facts_valid_complete(r) && py_facts_has_effect(r); + if (!ok) + printf(" control '%s' is no namespace effect\n", effects[i][0]); + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + } + PASS(); +} + TEST(python_namespace_facts_distinguish_all_operations_and_unknowns) { static const char source[] = "__all__ = ['old']; __all__ = []\n" "__all__ = dynamic\n" @@ -12796,6 +12906,7 @@ SUITE(extraction) { RUN_TEST(python_namespace_facts_parse_errors_are_complete_unknown); RUN_TEST(python_namespace_facts_type_checking_alternatives); RUN_TEST(python_namespace_facts_string_statements_and_future_imports); + RUN_TEST(python_namespace_facts_init_call_allowlist); RUN_TEST(python_namespace_facts_capture_and_copy_fail_atomically); RUN_TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema); RUN_TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 03cdb1933..2ee50f479 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14423,16 +14423,20 @@ static const char *const py_reexport_calls[][2] = { {"ctl_from_fn", "direct.impl.direct_fn"}, {"lazy_tc", "lz.lazy_mod.Lazy"}, {"doc_reexport", "docp.impl.Doc"}, + {"go_reexport", "gop.impl.Thing"}, }; enum { PY_REEXPORT_N_CALLS = sizeof(py_reexport_calls) / sizeof(py_reexport_calls[0]) }; /* Calls that must NOT reach their target by the LSP: an `__all__`-unlisted * name, a private name under a star, a re-export the module shadows later, - * and one made before a whole-namespace barrier (a top-level `try:`). */ + * one made before a whole-namespace barrier (a top-level `try:`), a + * re-export a submodule import of `__go` replaces, and one behind a `__go` + * that writes its parameter. */ static const char *const py_reexport_excluded[][2] = { {"excl_unlisted", "pkg.all_mod.Unlisted"}, {"excl_private", "pkg.star_mod._private_fn"}, {"excl_shadowed", "shad.impl.Gone"}, {"excl_barrier", "barr.impl.Before"}, {"excl_compat", "cmp.types_only.Shim"}, {"excl_fstring", "fsp.impl.Kept"}, + {"excl_go_submodule", "gop.impl.base"}, {"excl_go_param", "gox.impl.Kept"}, }; enum { PY_REEXPORT_N_EXCLUDED = sizeof(py_reexport_excluded) / sizeof(py_reexport_excluded[0]) }; @@ -14505,6 +14509,25 @@ static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *exc "from .impl import Kept\n" "f\"{Kept}\"\n"); write_temp_file(tmp, "app/fsp/impl.py", "class Kept:\n pass\n"); + /* sqlalchemy's sql/__init__.py shape: a top-level `__go(locals())`. */ + write_temp_file(tmp, "app/gop/__init__.py", + "from typing import Any\n" + "from .impl import Thing\n" + "from .impl import base\n\n\n" + "def __go(lcls: Any) -> None:\n" + " from . import base\n" + " from .annotation import prepare\n" + " prepare(Thing)\n\n\n" + "__go(locals())\n"); + write_temp_file(tmp, "app/gop/impl.py", + "class Thing:\n pass\n\n\ndef base():\n return 1\n"); + write_temp_file(tmp, "app/gop/base.py", "def helper():\n return 1\n"); + write_temp_file(tmp, "app/gop/annotation.py", "def prepare(cls):\n return cls\n"); + write_temp_file(tmp, "app/gox/__init__.py", + "from .impl import Kept\n\n\n" + "def __go(lcls):\n lcls[\"Kept\"] = None\n\n\n" + "__go(locals())\n"); + write_temp_file(tmp, "app/gox/impl.py", "class Kept:\n pass\n"); write_temp_file(tmp, "app/main.py", "from app import pkg\n" "from app import cyc\n" @@ -14514,6 +14537,8 @@ static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *exc "from app import cmp\n" "from app import docp\n" "from app import fsp\n" + "from app import gop\n" + "from app import gox\n" "from app.db import models\n" "from app.pkg import Explicit, Listed, Deep, star_fn, Renamed\n" "from app.direct.impl import Direct, direct_fn\n\n\n" @@ -14541,6 +14566,9 @@ static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *exc "def lazy_tc():\n lz.Lazy()\n\n\n" "def doc_reexport():\n docp.Doc()\n\n\n" "def excl_fstring():\n fsp.Kept()\n\n\n" + "def go_reexport():\n gop.Thing()\n\n\n" + "def excl_go_submodule():\n gop.base()\n\n\n" + "def excl_go_param():\n gox.Kept()\n\n\n" "def ctl_from_cls():\n Direct()\n\n\n" "def ctl_from_fn():\n direct_fn()\n"); for (int i = 0; parallel && i < 52; i++) { From 3281626c602a431e1047e4097104d000f4274d9f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 04:28:36 +0200 Subject: [PATCH 15/19] style(test): clang-format the __go allowlist test Signed-off-by: Martin Vogel --- tests/test_extraction.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index ebfa48f8c..d2f336400 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -12474,8 +12474,8 @@ TEST(python_namespace_facts_init_call_allowlist) { /* A plain module is no package: the body's imports set attributes of its * parent package, never of the module itself. */ - r = extract("def __go(lcls):\n from . import base\n\n__go(locals())\n", CBM_LANG_PYTHON, - "t", "sql/mod.py"); + r = extract("def __go(lcls):\n from . import base\n\n__go(locals())\n", CBM_LANG_PYTHON, "t", + "sql/mod.py"); ok = py_facts_valid_complete(r) && !py_facts_has_effect(r) && py_facts_count_kind(r, CBM_PY_NS_SHADOW) == 0; if (r) From 90bb6845d6304ae97f693d789f8df68888ef029d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 04:28:36 +0200 Subject: [PATCH 16/19] fix(python): a top-level call of a harmless same-module function keeps the namespace A top-level call withdrew the whole namespace (UNKNOWN, reason EFFECT): sqlalchemy's sql/__init__.py ends with __go(locals()) and lost all 108 re-exports. The call is now allowed when the callee is the latest binding of its name, a proven same-module function with no star import or unknown effect since; every argument is a safe expression or a bare locals()/globals() the module does not rebind; and the body (nested scopes included) has no global/nonlocal and never names a parameter, a function or class the module defined earlier, the module's own name, or a name that reaches a namespace (globals, vars, locals, exec, eval, setattr, delattr, __import__, sys, importlib, __name__, __dict__, __globals__, __builtins__, builtins, __self__, inspect, f_globals, f_locals, f_back, gc). Each submodule of the package the body imports becomes a package attribute and is shadowed at the call; in a plain module the imports install nothing. Every other call stays a whole-namespace effect. The function table and verdicts live in the extraction scratch; each callee is checked once. Signed-off-by: Martin Vogel --- internal/cbm/extract_imports.c | 526 ++++++++++++++++++++++++++++++++- 1 file changed, 515 insertions(+), 11 deletions(-) diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index b11436091..b6fda4135 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -484,12 +484,32 @@ static void parse_python_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } +/* A proven same-module function (its OWN_DEF fact and definition), and + * whether a top-level call of it leaves the namespace intact (see + * py_ns_allowed_call). */ +typedef enum { + PY_NS_CALL_UNCHECKED = 0, + PY_NS_CALL_ALLOWED = 1, + PY_NS_CALL_EFFECT = 2, +} PyNsCallVerdict; + +typedef struct { + int fact; + TSNode node; + PyNsCallVerdict verdict; + const char **shadows; /* submodules the body imports into the package */ + int shadow_count; +} PyNsFunction; + /* Ordered Python namespace facts are a separate content-only pass. The older * import records keep their existing graph/heuristic meaning. */ typedef struct { CBMExtractCtx *ctx; CBMPyNamespaceFacts facts; CBMPyNamespaceFailure failure; + PyNsFunction *functions; /* in fact order; scratch, never part of the result */ + int function_count; + int function_cap; } PyNamespaceBuilder; #ifdef CBM_ENABLE_TEST_SEAMS @@ -516,6 +536,18 @@ static void *py_ns_alloc(PyNamespaceBuilder *b, size_t size) { return p; } +/* Builder bookkeeping nothing in the result points at: the extraction scratch + * when the context has one. */ +static void *py_ns_scratch_alloc(PyNamespaceBuilder *b, size_t size) { + if (b->failure) + return NULL; + CBMArena *arena = b->ctx->scratch ? b->ctx->scratch : b->ctx->arena; + void *p = cbm_arena_alloc(arena, size); + if (!p) + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return p; +} + static const char *py_ns_copy(PyNamespaceBuilder *b, const char *text, size_t len) { if (len == SIZE_MAX) { b->failure = CBM_PY_NS_FAILURE_LIMIT; @@ -1024,6 +1056,472 @@ static bool py_ns_future_import(PyNamespaceBuilder *b, TSNode node) { return ok && emitted > 0; } +/* ── A top-level call the namespace survives ────────────────────────────── + * Any top-level call may rebind any name of the module, so it withdraws the + * whole namespace (UNKNOWN, reason EFFECT). sqlalchemy's sql/__init__.py ends + * with `__go(locals())`, which withdrew all of its 108 re-exports. A call is + * allowed when its callee provably cannot rebind the caller's names: + * - the callee is the latest binding of its name, a proven (undecorated) + * same-module function, with no star import or unknown effect since; + * - every argument is a safe expression or a bare `locals()` / `globals()` + * the module does not rebind; + * - the body, nested scopes included, has no `global` / `nonlocal` and never + * names a parameter, a function or class this module defined earlier (it + * could rebind through `global`), the module's own name, or a name that + * reaches a namespace (py_ns_namespace_names). + * The body may import: each submodule of this package it imports becomes a + * package attribute, so in a package `__init__` those names are shadowed at + * the call. Every other call stays a whole-namespace effect. */ + +static const char *const py_ns_namespace_names[] = { + "globals", "vars", "locals", "exec", "eval", "setattr", "delattr", + "__import__", "sys", "importlib", "__name__", "__dict__", "__globals__", "__builtins__", + "builtins", "__self__", "inspect", "f_globals", "f_locals", "f_back", "gc", +}; + +typedef struct { + const char *text; + size_t len; +} PyNsSlice; + +typedef struct { + PyNamespaceBuilder *b; + PyNsSlice own; + bool package; + PyNsSlice *params; + int param_count; + const char **defs; /* earlier same-module functions and classes, sorted */ + int def_count; + const char **shadows; + int shadow_count; + int shadow_cap; +} PyNsCallCheck; + +static bool py_ns_text_is(const char *text, size_t len, const char *s) { + return strlen(s) == len && memcmp(text, s, len) == 0; +} + +/* strcmp of a NUL-terminated string against a slice. */ +static int py_ns_slice_cmp(const char *s, const char *text, size_t len) { + int r = strncmp(s, text, len); + if (r != 0) + return r; + return s[len] ? 1 : 0; +} + +static int py_ns_cmp_cstr(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +/* `pkg/sub/__init__.py` -> "sub", a package; `pkg/mod.py` -> "mod". */ +static bool py_ns_own_name(const char *rel_path, PyNsSlice *own, bool *package) { + if (!rel_path) + return false; + const char *slash = strrchr(rel_path, '/'); + const char *file = slash ? slash + 1 : rel_path; + const char *ext = strrchr(file, '.'); + if (!ext || ext == file) + return false; + size_t stem = (size_t)(ext - file); + *package = py_ns_text_is(file, stem, "__init__"); + if (!*package) { + *own = (PyNsSlice){file, stem}; + } else { + if (!slash) + return false; + const char *dir = slash; + while (dir > rel_path && dir[-1] != '/') + dir--; + *own = (PyNsSlice){dir, (size_t)(slash - dir)}; + } + return py_ns_identifier_bytes(own->text, own->len, false); +} + +static bool py_ns_reaches_namespace(const char *text, size_t len) { + for (size_t i = 0; i < sizeof(py_ns_namespace_names) / sizeof(py_ns_namespace_names[0]); i++) { + if (py_ns_text_is(text, len, py_ns_namespace_names[i])) + return true; + } + return false; +} + +/* A name of the body (not a module path) the call cannot allow. */ +static bool py_ns_forbidden_name(const PyNsCallCheck *c, const char *text, size_t len) { + if (py_ns_reaches_namespace(text, len) || + (len == c->own.len && memcmp(text, c->own.text, len) == 0)) + return true; + for (int i = 0; i < c->param_count; i++) { + if (len == c->params[i].len && memcmp(text, c->params[i].text, len) == 0) + return true; + } + int lo = 0, hi = c->def_count - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + int r = py_ns_slice_cmp(c->defs[mid], text, len); + if (r == 0) + return true; + if (r < 0) + lo = mid + 1; + else + hi = mid - 1; + } + return false; +} + +static bool py_ns_node_forbidden(const PyNsCallCheck *c, TSNode node) { + const char *text = NULL; + size_t len = 0; + return ts_node_is_null(node) || !py_ns_slice(c->b, node, &text, &len) || + py_ns_forbidden_name(c, text, len); +} + +static bool py_ns_add_shadow(PyNsCallCheck *c, const char *text, size_t len) { + if (!py_ns_identifier_bytes(text, len, false)) + return false; + if (c->shadow_count == c->shadow_cap) { + int cap = c->shadow_cap ? c->shadow_cap * 2 : 8; + const char **items = py_ns_scratch_alloc(c->b, (size_t)cap * sizeof(*items)); + if (!items) + return false; + if (c->shadow_count) + memcpy(items, c->shadows, (size_t)c->shadow_count * sizeof(*items)); + c->shadows = items; + c->shadow_cap = cap; + } + const char *name = py_ns_copy(c->b, text, len); + if (!name) + return false; + c->shadows[c->shadow_count++] = name; + return true; +} + +/* The segments of a dotted module path: none may reach a namespace. Sets + * *own_at to the index of the segment naming this module (-1: none) and + * *next to the slice after it (empty when it is the last segment). */ +static bool py_ns_check_path(const PyNsCallCheck *c, const char *text, size_t len, int *own_at, + PyNsSlice *next) { + *own_at = -1; + *next = (PyNsSlice){NULL, 0}; + int index = 0; + size_t start = 0; + while (start <= len && len > 0) { + size_t end = start; + while (end < len && text[end] != '.') + end++; + const char *seg = text + start; + size_t seg_len = end - start; + if (py_ns_reaches_namespace(seg, seg_len)) + return false; + if (*own_at >= 0 && *own_at == index - 1) + *next = (PyNsSlice){seg, seg_len}; + if (*own_at < 0 && seg_len == c->own.len && memcmp(seg, c->own.text, seg_len) == 0) + *own_at = index; + index++; + start = end + 1; + } + return true; +} + +/* An import in the callee's body. When its module is this package, the + * submodule it installs as a package attribute is shadowed: the first + * segment after the package, or every imported member when the import names + * the package itself. A relative import of a package `__init__` is relative + * to the package (level 1) or to its parent (level 2, whose first segment + * then names this package or a sibling). Deeper relative imports that name + * this module cannot be placed and are refused. The names an import binds are + * checked like every name of the body; this module's own name is allowed in a + * module path only. */ +static bool py_ns_check_import(PyNsCallCheck *c, TSNode node, bool from) { + TSNode module = from ? ts_node_child_by_field_name(node, TS_FIELD("module_name")) : (TSNode){0}; + bool into_package = false; /* the import's module is this package */ + PyNsSlice sub = {NULL, 0}; /* the submodule after the package, if any */ + if (from) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(c->b, module, &text, &len)) + return false; + size_t level = 0; + while (level < len && text[level] == '.') + level++; + const char *rest = text + level; + size_t rest_len = len - level; + if (rest_len && !py_ns_identifier_bytes(rest, rest_len, true)) + return false; + int own_at = -1; + PyNsSlice next; + if (!py_ns_check_path(c, rest, rest_len, &own_at, &next)) + return false; + if (c->package && level == 1) { + into_package = rest_len == 0; + if (rest_len) { + size_t first = strcspn(rest, "."); + sub = (PyNsSlice){rest, first < rest_len ? first : rest_len}; + } + } else if (c->package && (level == 0 || (level == 2 && own_at == 0)) && own_at >= 0) { + /* level 2: the parent's child of this name is this package; a + * later segment of the same name is a package further down */ + if (next.len) + sub = next; + else + into_package = true; + } else if (c->package && level > 2 && own_at >= 0) { + return false; + } + if (sub.len && !py_ns_add_shadow(c, sub.text, sub.len)) + return false; + } + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || ts_node_eq(child, module) || + strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *kind = ts_node_type(child); + bool alias = strcmp(kind, "aliased_import") == 0; + if (!alias && strcmp(kind, "dotted_name") != 0) { + ok = false; /* a star import, or a form this check does not read */ + break; + } + TSNode name = alias ? ts_node_child_by_field_name(child, TS_FIELD("name")) : child; + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(c->b, name, &text, &len)) { + ok = false; + break; + } + if (alias && + py_ns_node_forbidden(c, ts_node_child_by_field_name(child, TS_FIELD("alias")))) { + ok = false; + break; + } + if (from) { + /* a member is a name the import binds (or a submodule) */ + if (py_ns_forbidden_name(c, text, len) || + (into_package && !py_ns_add_shadow(c, text, len))) { + ok = false; + break; + } + continue; + } + int own_at = -1; + PyNsSlice next; + if (!py_ns_identifier_bytes(text, len, true) || + !py_ns_check_path(c, text, len, &own_at, &next) || + (alias && own_at >= 0 && !next.len)) { + ok = false; /* `import pkg.own as alias` hands out this module */ + break; + } + if (c->package && own_at >= 0 && next.len && + !py_ns_add_shadow(c, next.text, next.len)) { + ok = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok; +} + +static bool py_ns_collect_params(PyNsCallCheck *c, TSNode def) { + TSNode params = ts_node_child_by_field_name(def, TS_FIELD("parameters")); + if (ts_node_is_null(params)) + return false; + uint32_t count = ts_node_named_child_count(params); + c->params = count ? py_ns_scratch_alloc(c->b, (size_t)count * sizeof(*c->params)) : NULL; + if (count && !c->params) + return false; + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(params, i); + const char *kind = ts_node_type(p); + if (strcmp(kind, "comment") == 0 || strcmp(kind, "keyword_separator") == 0 || + strcmp(kind, "positional_separator") == 0) + continue; + TSNode name = p; + if (strcmp(kind, "default_parameter") == 0 || + strcmp(kind, "typed_default_parameter") == 0) { + name = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(kind, "typed_parameter") == 0 || + strcmp(kind, "list_splat_pattern") == 0 || + strcmp(kind, "dictionary_splat_pattern") == 0) { + name = ts_node_named_child(p, 0); + const char *inner = ts_node_is_null(name) ? "" : ts_node_type(name); + if (strcmp(inner, "list_splat_pattern") == 0 || + strcmp(inner, "dictionary_splat_pattern") == 0) + name = ts_node_named_child(name, 0); + } else if (strcmp(kind, "identifier") != 0) { + return false; + } + PyNsSlice s; + if (ts_node_is_null(name) || strcmp(ts_node_type(name), "identifier") != 0 || + !py_ns_slice(c->b, name, &s.text, &s.len)) + return false; + c->params[c->param_count++] = s; + } + return true; +} + +static bool py_ns_collect_defs(PyNsCallCheck *c, int before) { + c->defs = before ? py_ns_scratch_alloc(c->b, (size_t)before * sizeof(*c->defs)) : NULL; + if (before && !c->defs) + return false; + for (int i = 0; i < before; i++) { + const CBMPyNamespaceFact *f = &c->b->facts.items[i]; + bool def = f->kind == CBM_PY_NS_OWN_DEF || + (f->kind == CBM_PY_NS_SHADOW && (f->reason == CBM_PY_NS_REASON_DECORATED || + f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS)); + if (def && f->local_name) + c->defs[c->def_count++] = f->local_name; + } + if (c->def_count > 1) + qsort(c->defs, (size_t)c->def_count, sizeof(*c->defs), py_ns_cmp_cstr); + return true; +} + +/* Walk the callee's body (iteratively: nesting must not exhaust the stack). */ +static bool py_ns_check_body(PyNsCallCheck *c, TSNode body) { + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool ok = true; + bool done = false; + while (ok && !done && !c->b->failure) { + TSNode node = ts_tree_cursor_current_node(&cursor); + const char *kind = ts_node_type(node); + bool from = strcmp(kind, "import_from_statement") == 0; + bool is_import = from || strcmp(kind, "import_statement") == 0; + if (strcmp(kind, "global_statement") == 0 || strcmp(kind, "nonlocal_statement") == 0 || + strcmp(kind, "future_import_statement") == 0) { + ok = false; + } else if (is_import) { + ok = py_ns_check_import(c, node, from); + } else if (strcmp(kind, "identifier") == 0) { + ok = !py_ns_node_forbidden(c, node); + } + if (!ok) + break; + if (!is_import && ts_tree_cursor_goto_first_child(&cursor)) + continue; + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (!ts_tree_cursor_goto_parent(&cursor)) { + done = true; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + return ok && !c->b->failure; +} + +static PyNsCallVerdict py_ns_check_function(PyNamespaceBuilder *b, PyNsFunction *fn) { + PyNsCallCheck c = {.b = b}; + TSNode body = ts_node_child_by_field_name(fn->node, TS_FIELD("body")); + if (ts_node_is_null(body) || !py_ns_own_name(b->ctx->rel_path, &c.own, &c.package) || + !py_ns_collect_params(&c, fn->node) || !py_ns_collect_defs(&c, fn->fact) || + !py_ns_check_body(&c, body)) + return PY_NS_CALL_EFFECT; + if (!c.package) + c.shadow_count = 0; /* a plain module's imports install nothing in it */ + fn->shadows = c.shadows; + fn->shadow_count = c.shadow_count; + return PY_NS_CALL_ALLOWED; +} + +static void py_ns_note_function(PyNamespaceBuilder *b, int fact, TSNode node) { + if (b->function_count == b->function_cap) { + int cap = b->function_cap ? b->function_cap * 2 : 16; + PyNsFunction *items = py_ns_scratch_alloc(b, (size_t)cap * sizeof(*items)); + if (!items) + return; + if (b->function_count) + memcpy(items, b->functions, (size_t)b->function_count * sizeof(*items)); + b->functions = items; + b->function_cap = cap; + } + b->functions[b->function_count++] = (PyNsFunction){.fact = fact, .node = node}; +} + +enum { PY_NS_UNBOUND = -1, PY_NS_BARRIER = -2 }; + +/* The index of the latest fact binding `name`: PY_NS_UNBOUND when none did, + * PY_NS_BARRIER when a star import or an unknown effect may have since. */ +static int py_ns_latest_binding(const PyNamespaceBuilder *b, const char *name, size_t len) { + for (int i = b->facts.count - 1; i >= 0; i--) { + const CBMPyNamespaceFact *f = &b->facts.items[i]; + if (f->kind == CBM_PY_NS_UNKNOWN || f->kind == CBM_PY_NS_IMPORT_STAR) + return PY_NS_BARRIER; + if (f->local_name && py_ns_slice_cmp(f->local_name, name, len) == 0) + return i; + } + return PY_NS_UNBOUND; +} + +/* Safe expressions, or a bare `locals()` / `globals()` of the builtins (the + * body never names the parameter that receives it). */ +static bool py_ns_call_arguments_safe(PyNamespaceBuilder *b, TSNode args) { + if (ts_node_is_null(args) || strcmp(ts_node_type(args), "argument_list") != 0) + return false; + uint32_t count = ts_node_named_child_count(args); + for (uint32_t i = 0; i < count; i++) { + TSNode arg = ts_node_named_child(args, i); + if (py_ns_safe_expr(b, arg, 0)) + continue; + if (strcmp(ts_node_type(arg), "call") != 0) + return false; + TSNode fn = ts_node_child_by_field_name(arg, TS_FIELD("function")); + TSNode inner = ts_node_child_by_field_name(arg, TS_FIELD("arguments")); + const char *text = NULL; + size_t len = 0; + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "identifier") != 0 || + !py_ns_slice(b, fn, &text, &len) || + !(py_ns_text_is(text, len, "locals") || py_ns_text_is(text, len, "globals")) || + ts_node_is_null(inner) || ts_node_named_child_count(inner) != 0 || + py_ns_latest_binding(b, text, len) != PY_NS_UNBOUND) + return false; + } + return true; +} + +/* True when the top-level call `call` cannot rebind a name of this module's + * namespace; the submodules its callee imports into the package are then + * shadowed here. False when it stays a whole-namespace effect. */ +static bool py_ns_allowed_call(PyNamespaceBuilder *b, TSNode call) { + TSNode callee = ts_node_child_by_field_name(call, TS_FIELD("function")); + const char *name = NULL; + size_t len = 0; + if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "identifier") != 0 || + !py_ns_slice(b, callee, &name, &len) || + !py_ns_call_arguments_safe(b, ts_node_child_by_field_name(call, TS_FIELD("arguments")))) + return false; + int at = py_ns_latest_binding(b, name, len); + if (at < 0 || b->facts.items[at].kind != CBM_PY_NS_OWN_DEF || + b->facts.items[at].def_kind != CBM_PY_NS_DEF_FUNCTION) + return false; + int lo = 0, hi = b->function_count - 1; + PyNsFunction *fn = NULL; + while (lo <= hi && !fn) { + int mid = lo + (hi - lo) / 2; + if (b->functions[mid].fact == at) + fn = &b->functions[mid]; + else if (b->functions[mid].fact < at) + lo = mid + 1; + else + hi = mid - 1; + } + if (!fn) + return false; + if (fn->verdict == PY_NS_CALL_UNCHECKED) + fn->verdict = py_ns_check_function(b, fn); + if (fn->verdict != PY_NS_CALL_ALLOWED || b->failure) + return false; + for (int i = 0; i < fn->shadow_count; i++) { + if (!py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = fn->shadows[i], + .reason = CBM_PY_NS_REASON_VALUE})) + return false; + } + return true; +} + static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *kind = ts_node_type(node); if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) @@ -1050,16 +1548,20 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) return false; } - return py_ns_push( - b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, - .local_name = name, - .def_kind = function ? CBM_PY_NS_DEF_FUNCTION - : CBM_PY_NS_DEF_CLASS} - : (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, - .local_name = name, - .reason = decorated - ? CBM_PY_NS_REASON_DECORATED - : CBM_PY_NS_REASON_UNPROVEN_CLASS}); + if (!py_ns_push( + b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, + .local_name = name, + .def_kind = function ? CBM_PY_NS_DEF_FUNCTION + : CBM_PY_NS_DEF_CLASS} + : (CBMPyNamespaceFact){ + .kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = decorated ? CBM_PY_NS_REASON_DECORATED + : CBM_PY_NS_REASON_UNPROVEN_CLASS})) + return false; + if (proven && function) + py_ns_note_function(b, b->facts.count - 1, def); + return !b->failure; } } if (strcmp(kind, "delete_statement") == 0) @@ -1069,7 +1571,9 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *ek = ts_node_type(expr); if (strcmp(ek, "assignment") == 0 || strcmp(ek, "augmented_assignment") == 0) return py_ns_assignment(b, expr); - return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); + if (py_ns_safe_expr(b, expr, 0) || (strcmp(ek, "call") == 0 && py_ns_allowed_call(b, expr))) + return true; + return py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); } if (strcmp(kind, "if_statement") == 0 && py_ns_is_type_checking_if(b, node)) return py_ns_type_checking_block(b, node); From 1e644b4414544a3da6d0ee10a6cb5121c78af61a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 04:55:07 +0200 Subject: [PATCH 17/19] fix(python): bound the module-path scan of the __go import check WHY: the check read an import's module path with strcspn, which runs to a NUL, but a tree-sitter source slice need not be NUL-terminated: the first segment is now found with memchr inside the slice. The leading dots move into py_ns_leading_dots; written inline, cppcheck 2.20 (the CI version) took the loop to always end at the slice length and reported three knownConditionTrueFalse findings on the branches that follow it. Signed-off-by: Martin Vogel --- internal/cbm/extract_imports.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index b6fda4135..c19debd80 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -1222,6 +1222,15 @@ static bool py_ns_check_path(const PyNsCallCheck *c, const char *text, size_t le return true; } +/* The relative level of a `from` module path: its leading dots. */ +static size_t py_ns_leading_dots(const char *text, size_t len) { + const char *end = text + len; + const char *p = text; + while (p != end && *p == '.') + p++; + return (size_t)(p - text); +} + /* An import in the callee's body. When its module is this package, the * submodule it installs as a package attribute is shadowed: the first * segment after the package, or every imported member when the import names @@ -1240,23 +1249,19 @@ static bool py_ns_check_import(PyNsCallCheck *c, TSNode node, bool from) { size_t len = 0; if (!py_ns_slice(c->b, module, &text, &len)) return false; - size_t level = 0; - while (level < len && text[level] == '.') - level++; - const char *rest = text + level; - size_t rest_len = len - level; - if (rest_len && !py_ns_identifier_bytes(rest, rest_len, true)) + size_t level = py_ns_leading_dots(text, len); + PyNsSlice rest = {text + level, len - level}; + if (rest.len && !py_ns_identifier_bytes(rest.text, rest.len, true)) return false; int own_at = -1; PyNsSlice next; - if (!py_ns_check_path(c, rest, rest_len, &own_at, &next)) + if (!py_ns_check_path(c, rest.text, rest.len, &own_at, &next)) return false; if (c->package && level == 1) { - into_package = rest_len == 0; - if (rest_len) { - size_t first = strcspn(rest, "."); - sub = (PyNsSlice){rest, first < rest_len ? first : rest_len}; - } + /* `from . import x` names the package; `from .x.y import z` installs x */ + const char *dot = rest.len ? memchr(rest.text, '.', rest.len) : NULL; + into_package = rest.len == 0; + sub = (PyNsSlice){rest.text, dot ? (size_t)(dot - rest.text) : rest.len}; } else if (c->package && (level == 0 || (level == 2 && own_at == 0)) && own_at >= 0) { /* level 2: the parent's child of this name is this package; a * later segment of the same name is a package further down */ From 5cdbe9b0b664de1ba0cdb004e696b2d7bf153061 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 23:42:36 +0200 Subject: [PATCH 18/19] fix(deploy): taint Helm changes of and/or, later conditions, sortAlpha, parse errors Review round 4 of #2677 (user decisions: R4-1..R4-4, R4-6, R4-7, R4-9; R4-5 measured and left out). Every case ends in an UNKNOWN image, never a wrong one. Changes not run exactly (helm_template.c): - A deep merge into a values map is not exact when a change was made at or below its path (a set storing a map there): every values map a set or a merge changes keeps its path, and such a merge leaves the images after it UNKNOWN. - and / or at an operand whose truth is not known -- UNKNOWN, or a map a change made unknown -- leave the images after them UNKNOWN when an operand after it may change maps. - In an if / with an UNKNOWN condition skips, the else-if / else-with conditions after it are looked at too: a change there taints, an assignment there makes its variable UNKNOWN. - sortAlpha sorts a []string in place: it counts as a change (a list is never poisoned, so it taints). No other Sprig list function changes its argument in place. - A template, a define body or a scanned template file the grammar does not parse may hide a change: its images are UNKNOWN. Work (helm_template.c): - A capture's keys leave the key set one by one; the set is no longer cleared by walking its whole capacity after each capture. - Template files are found by a hash of their path, and an include of a name that is no define is answered once per render. Known limit, documented (README): each template file renders on its own, while Helm renders a chart's files with one shared .Values. Running them in Helm's order is the planned follow-up; the coarse rule across files cost 83 more correct images on the 20 infra corpora and was left out. On the 20 infra corpora: Image edges identical to round 4 (0 lost, 0 added), no budget trip. Tests: R4-1..R4-4 and R4-9 fail on 3401def5 by their assertions (each sub case too); R4-6 and R4-7 are correctness controls, their cost measured outside the suite. Signed-off-by: Martin Vogel --- README.md | 1 + src/pipeline/helm_template.c | 172 +++++++++++++++++++++++++--------- tests/test_grammar_probe_f.c | 9 +- tests/test_pipeline.c | 177 +++++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 590c9eef9..ee1db98bb 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,7 @@ The install script placed beside the binary is **reported, not deleted**: uninst - `INFRA_MAPS`: a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace - `CONNECTS_TO`: a Kubernetes workload or compose service to the Service its containers' environment addresses (`PRODUCT_CATALOG_SERVICE_ADDR: productcatalogservice:3550`, a URL's host, `..svc...`, a bare name on an `*_ADDR` / `*_HOST` / `*_URL` / ... key; compose: the project's `.env` interpolated): a Kubernetes Service of the same bundle and namespace, or a compose service of the same file. The edge keeps `via: env`, `key` and `container`, never the value - `USAGE` with `ref: image`: a Kubernetes workload (any pod template: containers, initContainers, CronJobs, custom resources) or a docker compose service to the `Image` it runs +- Helm templates are rendered for their images: with the chart's values (the `Chart` `DEPLOYS` them, `default_of`: the values keys read) and with each release's (a Flux HelmRelease, a kustomize `helmCharts` entry, a Terraform `helm_release`: `override_of`). An image the render cannot know is left out, never guessed: after a change of values it does not run exactly, a `tpl` or include it cannot look into, a release value it cannot read, or a work budget it runs past. Known limit: each template file renders on its own, while Helm renders a chart's files with one shared `.Values` (a subchart's is its parent's `.Values.`), so a chart that changes `.Values` in one file and reads it in another can still show a wrong image - `DEPLOYS`: what puts manifests, kustomizations or charts on a cluster, to what it puts there: a Flux Kustomization or Argo Application to the kustomizations / manifests / chart of its path, a Flux HelmRelease to its chart (`spec.chart` or `chartRef` → OCIRepository), a shell command or Make recipe (`kubectl apply -f/-k`, `kustomize build | kubectl apply`, `helm install/upgrade/template`), a Go installer loading a chart with the Helm SDK, a Terraform `helm_release`, a Terraform `kubernetes_*` resource the object it declares (a `Resource` like a manifest's, `ecosystem: terraform`, when its name is a literal), a kustomization inflating a chart (`helmCharts`; its values file an `IMPORTS`). A chart outside the repository is an external `Chart`, keyed by its publisher when its repository URL is stated (`__chart__/`, the same node in every project, the URL in `repository`), else one per project and name; the edge says `via` (`flux`, `argo`, `helmrelease`, `kubectl`, `kustomize`, `helm`, `helm-sdk`, `terraform`, `helmcharts`, ...) - `DEPENDS_ON`: a Helm chart to the charts it declares (the local chart for a resolvable `file://` repository, otherwise an external `Chart`; the edge keeps `repository`), and a `go.mod` / `requirements.txt` file to an external `Package`; a Dockerfile to the `Image`s it builds on (`via: from`, `copy-from`); an `Image` to the Dockerfile it is built from (`via: build`; `tool`: `skaffold`, `compose`, `docker-build` for `docker` / `podman` / `nerdctl` / `buildah` build commands in scripts, Make and CI); a compose service to the services its `depends_on` / `links` name - `CONFIGURES`: code reading an environment variable to the shared `EnvVar` of that name (`strategy: env_access`); a Kubernetes container, compose service or Dockerfile (`ENV` of the image) setting one to that container's own `EnvVar` (`strategy: env_set`, with `source`: `value`, `configMapKeyRef`, `secretKeyRef`, `fieldRef`, ..., `compose`, `dockerfile`; values are never stored); and that container's `EnvVar` to the code reading the name inside the files its image is built from (`strategy: env_read`, with `dockerfile`): the image's files are the sources its Dockerfile's `COPY` / `ADD` put into it, less the context's `.dockerignore` diff --git a/src/pipeline/helm_template.c b/src/pipeline/helm_template.c index 5004c2e7d..7c9a5d894 100644 --- a/src/pipeline/helm_template.c +++ b/src/pipeline/helm_template.c @@ -916,15 +916,17 @@ struct cbm_helm_defines { hd_file_t **files; /* each allocated once: defines point into them */ int nfiles; int cap; - CBMHashTable *names; /* name -> hd_def_t* */ + CBMHashTable *names; /* name -> hd_def_t* */ + CBMHashTable *by_rel; /* path -> hd_file_t* */ }; cbm_helm_defines_t *cbm_helm_defines_new(void) { cbm_helm_defines_t *d = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*d)); if (d) { d->names = cbm_ht_create(0); - if (!d->names) { - cbm_free(CBM_MEM_CLASS_OTHER, d); + d->by_rel = cbm_ht_create(0); + if (!d->names || !d->by_rel) { + cbm_helm_defines_free(d); return NULL; } } @@ -1018,6 +1020,7 @@ void cbm_helm_defines_add(cbm_helm_defines_t *d, const char *rel, const char *sr ts_parser_delete(parser); } d->files[d->nfiles++] = f; + cbm_ht_set(d->by_rel, f->rel, f); if (f->tree) { hd_scan(d, f, ts_tree_root_node(f->tree), hd_depth(rel)); } @@ -1037,6 +1040,9 @@ void cbm_helm_defines_free(cbm_helm_defines_t *d) { cbm_ht_foreach(d->names, hd_free_def, NULL); cbm_ht_free(d->names); } + if (d->by_rel) { + cbm_ht_free(d->by_rel); /* its keys are the files' own */ + } for (int i = 0; i < d->nfiles; i++) { if (d->files[i]->tree) { ts_tree_delete(d->files[i]->tree); @@ -1051,12 +1057,7 @@ void cbm_helm_defines_free(cbm_helm_defines_t *d) { /* Template file `rel` of the set; NULL: none. */ static const hd_file_t *hd_file_find(const cbm_helm_defines_t *d, const char *rel) { - for (int i = 0; d && i < d->nfiles; i++) { - if (strcmp(d->files[i]->rel, rel) == 0) { - return d->files[i]; - } - } - return NULL; + return d && d->by_rel ? cbm_ht_get(d->by_rel, rel) : NULL; } static const hd_def_t *hd_find(const cbm_helm_defines_t *d, const char *name) { @@ -1394,6 +1395,11 @@ typedef struct { * and for a merge of values also by values path (everything below it) */ CBMHashTable *poison_maps; CBMHashTable *poison_paths; + /* the values path of each values map changed (a set or a merge): a merge + * above one of them reaches what it put there */ + const char **changes; + int nchanges; + int chcap; /* a change of maps not run exactly: every capture ending later is UNKNOWN */ bool tainted; CBMHashTable *mutates; /* define name -> it may change a map (he_define_may_mutate) */ @@ -1655,6 +1661,37 @@ static bool hst_poison(hst_t *st, const hv_t *v, bool deep) { return hst_poison_at(st, v, deep, 0); } +/* Has a values map at path p or below it been changed? */ +static bool hst_changed_below(hst_t *st, const char *p) { + size_t lp = strlen(p); + for (int i = 0; i < st->nchanges; i++) { + const char *c = st->changes[i]; + if (!hst_step(st) || + (strncmp(c, p, lp) == 0 && (c[lp] == '\0' || c[lp] == '.' || c[lp] == '['))) { + return true; + } + } + return false; +} + +/* Values map at path p changed in place; false with no room to say so. */ +static bool hst_record_change(hst_t *st, const char *p) { + if (st->nchanges == st->chcap) { + int cap = st->chcap ? st->chcap * PAIR_LEN : CBM_SZ_16; + const char **g = cbm_arena_alloc(st->a, (size_t)cap * sizeof(*g)); + if (!g) { + return false; + } + if (st->nchanges) { + memcpy(g, st->changes, (size_t)st->nchanges * sizeof(*g)); + } + st->changes = g; + st->chcap = cap; + } + st->changes[st->nchanges++] = p; + return true; +} + static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { if (depth > HELM_MAX_NESTING || !hst_step(st)) { return false; @@ -1666,9 +1703,12 @@ static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { return false; } /* a deep change reaches every map within: one not known, one a change - * already made unknown, the values root (every path below it), or a map - * not of the values (whose maps no path covers) cannot be poisoned whole */ - if (deep && (hv_poisoned(st, v) || (v->path && !v->path[0]) || (v->k == HV_JSON && !v->path))) { + * already made unknown, the values root (every path below it), a map not + * of the values (whose maps no path covers), or a values map below which + * a change was made (it may hold another's map now) cannot be poisoned + * whole */ + if (deep && (hv_poisoned(st, v) || (v->path && !v->path[0]) || (v->k == HV_JSON && !v->path) || + (v->path && hst_changed_below(st, v->path)))) { return false; } if (deep && v->k == HV_MAP) { @@ -1681,7 +1721,7 @@ static bool hst_poison_at(hst_t *st, const hv_t *v, bool deep, int depth) { } char key[CBM_SZ_32]; snprintf(key, sizeof(key), "%p", map); - if (!hst_set_add(st, &st->poison_maps, key)) { + if (!hst_set_add(st, &st->poison_maps, key) || (v->path && !hst_record_change(st, v->path))) { return false; } return !(deep && v->path) || hst_set_add(st, &st->poison_paths, v->path); @@ -1935,12 +1975,15 @@ static void hst_capture_end(hst_t *st) { p->keys[p->n] = whole ? keys : NULL; p->n++; } + /* the keys this capture recorded, one by one: a clear walks the set's + * whole capacity, which one huge capture leaves behind for every later + * one */ + for (int i = 0; st->ckset && i < st->nck; i++) { + (void)cbm_ht_delete(st->ckset, st->ckeys[i]); + } st->capture = -SKIP_ONE; st->cap = (hsink_t){0}; st->nck = 0; - if (st->ckset) { - cbm_ht_clear(st->ckset); - } } /* Output of the top-level template: into the capture until its newline. */ @@ -2415,10 +2458,16 @@ static hv_t he_strfn(hst_t *st, const char *fn, const hv_t *a, int n) { return HV_UNKNOWN; } -/* Sprig functions that change a map in place. */ +/* Sprig functions that change their argument in place: set / unset / merge + * a map, sortAlpha a []string (splitList, keys, regexSplit give one; a list + * is never poisoned, so it always counts as a change not run). Every other + * list function copies its argument (push, append, prepend, reverse, uniq, + * without, compact, rest, initial, concat, chunk) or reads it (first, last, + * has, slice -- which shares it but changes nothing). */ static bool he_mutator(const char *s, size_t n) { static const char *const MUTATORS[] = { - "set", "unset", "merge", "mergeOverwrite", "mustMerge", "mustMergeOverwrite", NULL}; + "set", "unset", "merge", "mergeOverwrite", "mustMerge", "mustMergeOverwrite", + "sortAlpha", NULL}; return hy_in(s, n, MUTATORS); } @@ -2478,8 +2527,8 @@ static bool he_is_fn(const char *s, size_t n, const char *name) { * looked into), or an include / template of a name not known or of one that * may (he_define_may_mutate). A variable's name ($set) is no call. */ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth) { - if (depth > HELM_MAX_NESTING) { - return true; + if (depth > HELM_MAX_NESTING || ts_node_has_error(n)) { + return true; /* a form the grammar rejects (Go may not) may hide a change */ } TSTreeCursor cur = ts_tree_cursor_new(n); bool found = false; @@ -2514,6 +2563,9 @@ static bool he_may_mutate(hst_t *st, const char *src, TSNode n, int depth) { return found || st->stop; } +/* Memo states of he_define_may_mutate / he_file_may_mutate. */ +enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3, HM_STALE = 4 }; + /* The chart's template file an include names -- Helm names each one * `<.Template.BasePath>/` -- or NULL: none of this chart's. */ static const hd_file_t *he_template_file(hst_t *st, const char *name) { @@ -2534,7 +2586,6 @@ static const hd_file_t *he_template_file(hst_t *st, const char *name) { * while it is being scanned -- one including itself -- may. A no that leaned * on a define still being scanned is not kept. */ static bool he_file_may_mutate(hst_t *st, const hd_file_t *f, int depth) { - enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3 }; if (!f->tree) { return true; } @@ -2564,14 +2615,6 @@ static bool he_file_may_mutate(hst_t *st, const hd_file_t *f, int depth) { * scanned again when next asked), so defines including each other never fix * a wrong no -- a yes is final. */ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { - enum { HM_NO = 1, HM_YES = 2, HM_BUSY = 3, HM_STALE = 4 }; - const hd_def_t *d = hd_find(st->defs, name); - if (!d) { - /* a template file of the chart by its name (the checksum idiom) is - * looked into; a name not the chart's (a library's) may */ - const hd_file_t *f = he_template_file(st, name); - return f ? he_file_may_mutate(st, f, depth) : true; - } if (!st->mutates) { st->mutates = cbm_ht_create(0); } @@ -2579,6 +2622,21 @@ static bool he_define_may_mutate(hst_t *st, const char *name, int depth) { if (memo == HM_YES || memo == HM_NO) { return memo == HM_YES; } + const hd_def_t *d = hd_find(st->defs, name); + if (!d) { + /* a template file of the chart by its name (the checksum idiom) is + * looked into; a name not the chart's (a library's) may. Kept by + * name once final, so an include in a loop asks once */ + const hd_file_t *f = he_template_file(st, name); + bool m = f ? he_file_may_mutate(st, f, depth) : true; + bool final = + m || (st->file_mutates && (uintptr_t)cbm_ht_get(st->file_mutates, f->rel) == HM_NO); + const char *key = final && st->mutates ? cbm_arena_strdup(st->a, name) : NULL; + if (key) { + cbm_ht_set(st->mutates, key, (void *)(uintptr_t)(m ? HM_YES : HM_NO)); + } + return m; + } if (memo == HM_BUSY) { st->busy_hits++; return false; @@ -2988,7 +3046,15 @@ static hv_t he_call(hst_t *st, hframe_t *fr, TSNode call, const hv_t *piped) { uint32_t total = na + (piped ? SKIP_ONE : 0); for (uint32_t i = 0; i < total; i++) { v = i < na ? he_eval(st, fr, ts_node_named_child(args, i)) : *piped; - if (v.k == HV_UNK) { + if (v.k == HV_UNK || hv_poisoned(st, &v)) { + /* its truth is not known (a poisoned map's either): whether + * the operands after it run is not, nor a change they make */ + for (uint32_t j = i + SKIP_ONE; j < na; j++) { + if (he_may_mutate(st, fr->src, ts_node_named_child(args, j), st->nesting)) { + hst_taint(st, fn); + break; + } + } return HV_UNKNOWN; } bool t = hv_truth(&v); @@ -3212,20 +3278,25 @@ static int he_scan_skipped(hst_t *st, hframe_t *fr, TSNode n, bool in_range) { return leave; } -/* The children of n but its condition / collection (what was run), each: +/* The children of n but what was run -- its collection, its conditions up + * to `ran` (the one run; an else if / else with after it would be) -- each: * the variables it would assign UNKNOWN, and a change of maps it may make * (he_may_mutate) leaves every image after it UNKNOWN; returns how it would * leave the loop. */ -static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, const char *only, bool in_range) { +static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, TSNode ran, const char *only, + bool in_range) { int leave = HL_NONE; bool mutates = false; + bool after = false; /* past `ran` */ uint32_t cc = ts_node_child_count(n); for (uint32_t i = 0; i < cc; i++) { TSNode c = ts_node_child(n, i); const char *f = ts_node_field_name_for_child(n, i); - if (!ts_node_is_named(c) || hn_is(c, "range_variable_definition") || - (f && (strcmp(f, "condition") == 0 || strcmp(f, "range") == 0)) || - (only && !(f && strcmp(f, only) == 0))) { + bool cond = f && strcmp(f, "condition") == 0; + bool was_run = cond && !after; + after = after || (cond && ts_node_eq(c, ran)); + if (!ts_node_is_named(c) || hn_is(c, "range_variable_definition") || was_run || + (f && strcmp(f, "range") == 0) || (only && !(f && strcmp(f, only) == 0))) { continue; } leave |= he_scan_skipped(st, fr, c, in_range); @@ -3237,16 +3308,17 @@ static int he_spoil_parts(hst_t *st, hframe_t *fr, TSNode n, const char *only, b return leave; } -/* The bodies of if / with / range n, not run because its condition or - * collection is UNKNOWN (that one was run): their output, the variables they - * would assign and a change they would make are UNKNOWN. A break / continue - * in them may or may not leave the loop around: what the rest of that loop's - * body would assign or change is UNKNOWN too, and a break leaves it now (its - * later iterations are not known), a continue this iteration. */ -static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { +/* The bodies of if / with / range n, not run because its condition `ran` or + * its collection (a null `ran`) is UNKNOWN (that one was run): their output, + * the variables they -- and the conditions after `ran` -- would assign and a + * change they would make are UNKNOWN. A break / continue in them may or may + * not leave the loop around: what the rest of that loop's body would assign + * or change is UNKNOWN too, and a break leaves it now (its later iterations + * are not known), a continue this iteration. */ +static void he_skipped(hst_t *st, hframe_t *fr, TSNode n, TSNode ran) { hst_out_unknown(st, fr); bool range = hn_is(n, "range_action"); - int leave = he_spoil_parts(st, fr, n, NULL, range); + int leave = he_spoil_parts(st, fr, n, ran, NULL, range); if (leave == HL_NONE) { return; } @@ -3257,7 +3329,7 @@ static void he_skipped(hst_t *st, hframe_t *fr, TSNode n) { loop = ts_node_parent(loop); } if (hn_is(loop, "range_action")) { - (void)he_spoil_parts(st, fr, loop, "body", true); + (void)he_spoil_parts(st, fr, loop, (TSNode){0}, "body", true); } if (leave & HL_BREAK) { st->brk = true; @@ -3293,7 +3365,7 @@ static void he_branches(hst_t *st, hframe_t *fr, TSNode n, bool with) { if (v.k == HV_UNK || hv_poisoned(st, &v)) { stop = true; active = false; - he_skipped(st, fr, n); /* which branch renders is unknown */ + he_skipped(st, fr, n, c); /* which branch renders is unknown */ continue; } active = hv_truth(&v); @@ -3331,7 +3403,7 @@ static void he_range(hst_t *st, hframe_t *fr, TSNode n) { } hv_t coll = he_eval(st, fr, expr); if (coll.k == HV_UNK || hv_poisoned(st, &coll)) { - he_skipped(st, fr, n); + he_skipped(st, fr, n, (TSNode){0}); return; } hv_t *items = NULL; @@ -3472,6 +3544,9 @@ static hv_t he_render_define(hst_t *st, const char *name, hv_t arg, hsink_t *int hst_taint(st, "include"); /* nested too deep: not run */ return HV_UNKNOWN; } + if (ts_node_has_error(d->node)) { + hst_taint(st, "parse"); /* a form the grammar rejects may hide a change */ + } int mark = st->nvars; hframe_t sub = {.src = d->file->src, .dot = arg, .var_base = st->nvars, .out = into}; hst_var_push(st, "", 0, arg); @@ -3584,6 +3659,11 @@ bool cbm_helm_render(CBMArena *a, const char *rel, const char *src, size_t len, uint32_t *lines = cbm_arena_alloc(&work, (size_t)nl * sizeof(*lines)); TSTree *tree = lines ? ts_parser_parse_string(st.parser, NULL, src, (uint32_t)len) : NULL; if (tree) { + /* a template that does not parse may hide a change (Go may take a + * form the grammar rejects): no image of it is known */ + if (ts_node_has_error(ts_tree_root_node(tree))) { + hst_taint(&st, "parse"); + } int k = 0; lines[k++] = 0; for (size_t i = 0; i < len; i++) { diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 0c8cf008e..b51ad995e 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -4947,7 +4947,14 @@ TEST(probe_helm_release_terraform_postrender_forms) { * app/charts/cache/templates, as Helm sets it (an image here shows it). Its * checksum of cm.yaml names its own file, which changes no map -- not app's * cm.yaml, which sets a value: its image stays known, as does app's (which - * checksums its Secret). */ + * checksums its Secret). + * + * NOT a cross-file guarantee: app's cm.yaml sets .Values.image.tag, and Helm + * renders a chart's files with one shared .Values (a subchart's is the + * parent's .Values.). Each file renders alone here, so app's image is + * known only because nothing models that set reaching deploy.yaml -- the + * known limit (review round 4, R4-5), whose exact fix is to run the family's + * files in Helm's order with one .Values (the run-included-files follow-up). */ TEST(probe_helm_checksum_include_by_base_path) { static const GpfFile files[] = { {"charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 1.0.0\ndependencies:\n" diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index aef677834..f004cca3e 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5050,6 +5050,176 @@ TEST(helm_template_file_including_itself) { PASS(); } +static const char HELM_IMAGE_VALUES[] = + "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\ncfg: {a: {}}\nempty: {}\n"; +#define HELM_IMAGE_LINE "image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n" + +/* A set below a values map puts .Values.image at cfg.a.d; a deep merge into + * the map above it (.Values.cfg) reaches .Values.image there: the image after + * it is UNKNOWN. */ +TEST(helm_deep_merge_over_a_change_below) { + static const char tpl[] = + "{{- $_ := set .Values.cfg.a \"d\" .Values.image }}\n" + "{{- $_ := mergeOverwrite .Values.cfg (dict \"a\" (dict \"d\" (dict \"tag\" \"2.0\"))) }}\n" + HELM_IMAGE_LINE; + char *got = helm_capture(HELM_IMAGE_VALUES, tpl, NULL, 3); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* and / or stop at an operand whose truth is not known (.Release.Namespace; + * a map a set changed -- .Values.empty is empty no more): whether the + * operands after it run is not known, nor a change they make -- the image + * after it is UNKNOWN. A known false one runs none: the image stays known. */ +TEST(helm_and_or_with_an_unknown_operand) { + static const char *const tpls[] = { + "{{- $_ := and .Release.Namespace (set .Values.image \"tag\" \"2.0\") }}\n" HELM_IMAGE_LINE, + "{{- $_ := or .Release.Namespace (set .Values.image \"tag\" \"2.0\") }}\n" HELM_IMAGE_LINE, + "{{- $_ := set .Values.empty \"x\" \"y\" }}{{- $_ := and .Values.empty " + "(set .Values.image \"tag\" \"2.0\") }}\n" HELM_IMAGE_LINE}; + for (size_t i = 0; i < sizeof(tpls) / sizeof(tpls[0]); i++) { + char *got = helm_capture(HELM_IMAGE_VALUES, tpls[i], NULL, 2); + bool unknown = strcmp(got, "") == 0; + free(got); + ASSERT_TRUE(unknown); + } + static const char known[] = + "{{- $_ := and false (set .Values.image \"tag\" \"2.0\") }}\n" HELM_IMAGE_LINE; + char *got = helm_capture(HELM_IMAGE_VALUES, known, NULL, 2); + ASSERT_STR_EQ(got, "ghcr.io/acme/app:1.0"); + free(got); + PASS(); +} + +/* An if whose first condition is UNKNOWN: Helm evaluates the else-if + * conditions after it -- a set there, or an assignment ($tag = ...), may run: + * the image after it is UNKNOWN. */ +TEST(helm_later_condition_of_a_skipped_if) { + static const char set[] = + "{{- if regexMatch \"^v\" .Values.image.tag }}{{- else if set .Values.image \"tag\" \"2.0\" }}" + "{{- end }}\n" HELM_IMAGE_LINE; + char *got = helm_capture(HELM_IMAGE_VALUES, set, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + static const char assign[] = + "{{- $tag := \"1.0\" }}{{- if regexMatch \"^v\" .Values.image.tag }}" + "{{- else if $tag = \"2.0\" }}{{- end }}\n" + "image: \"ghcr.io/acme/app:{{ $tag }}\"\n"; + got = helm_capture(HELM_IMAGE_VALUES, assign, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* sortAlpha sorts a []string in place (splitList gives one): the list $tags + * holds changes -- not run here, so the image after it is UNKNOWN. */ +TEST(helm_sort_alpha_changes_its_list) { + static const char tpl[] = "{{- $tags := splitList \",\" .Values.tags }}" + "{{- $_ := sortAlpha $tags }}\n" + "image: \"ghcr.io/acme/app:{{ first $tags }}\"\n"; + char *got = helm_capture("tags: \"2.0,1.0\"\n", tpl, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + PASS(); +} + +/* A form the grammar rejects may hide a change (Go may take it): a template, + * a define it runs, or a template file it includes that does not parse + * leaves its images UNKNOWN. */ +TEST(helm_parse_error_spoils) { + static const char bad[] = "{{- $x := }}\n" HELM_IMAGE_LINE; + char *got = helm_capture(HELM_IMAGE_VALUES, bad, NULL, 2); + ASSERT_STR_EQ(got, ""); + free(got); + static const char defs_src[] = + "{{- define \"bad\" -}}{{- $x := }}{{- end -}}\n" + "{{- define \"good\" -}}ok{{- end -}}\n"; + cbm_helm_defines_t *d = cbm_helm_defines_new(); + ASSERT_NOT_NULL(d); + cbm_helm_defines_add(d, "charts/app/templates/_bad.tpl", defs_src, strlen(defs_src)); + static const char bad_file[] = "{{- $x := }}\nkind: ConfigMap\n"; + cbm_helm_defines_add(d, "charts/app/templates/bad.yaml", bad_file, strlen(bad_file)); + static const char good[] = "x: {{ include \"good\" . }}\n" HELM_IMAGE_LINE; + static const char via_define[] = "x: {{ include \"bad\" . }}\n" HELM_IMAGE_LINE; + static const char via_file[] = + "x: {{ include (print $.Template.BasePath \"/bad.yaml\") . | sha256sum }}\n" HELM_IMAGE_LINE; + char *ok = helm_capture_chart(HELM_IMAGE_VALUES, good, d, &HELM_APP, 2); + char *by_define = helm_capture_chart(HELM_IMAGE_VALUES, via_define, d, &HELM_APP, 2); + char *by_file = helm_capture_chart(HELM_IMAGE_VALUES, via_file, d, &HELM_APP, 2); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(ok, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(by_define, ""); + ASSERT_STR_EQ(by_file, ""); + free(ok); + free(by_define); + free(by_file); + PASS(); +} + +/* One capture reading 20,000 keys, then 2,000 captures of two keys each: + * every image and mark after the big one stays right (its keys leave the + * set one by one; the set is never cleared by walking its capacity). */ +TEST(helm_captures_after_a_huge_capture) { + enum { ITEMS = 20000, LATER = 2000 }; + size_t cap = (size_t)ITEMS * 8 + CBM_SZ_256; + char *vals = malloc(cap); + ASSERT_NOT_NULL(vals); + size_t o = (size_t)snprintf(vals, cap, "image: {repository: ghcr.io/acme/app, tag: \"1.0\"}\n" + "big: ["); + for (int i = 0; i < ITEMS; i++) { + o += (size_t)snprintf(vals + o, cap - o, "%s{x: a}", i ? "," : ""); + } + snprintf(vals + o, cap - o, "]\n"); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *v = doc ? cbm_helm_values_parse(doc, vals, strlen(vals)) : NULL; + free(vals); + static const char tpl[] = + "image: \"r/{{ range .Values.big }}{{ .x }}{{ end }}\"\n" + "{{- range 2000 }}\n" + "image: \"{{ $.Values.image.repository }}:{{ $.Values.image.tag }}\"\n" + "{{- end }}\n"; + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_4K); + cbm_helm_probe_t probes[2] = {{.line = 1}, {.line = 3}}; + bool ok = cbm_helm_render(&a, "t.yaml", tpl, strlen(tpl), v, NULL, NULL, probes, 2); + int n = probes[1].n; + bool last = n == LATER && probes[1].image[n - 1] && + strcmp(probes[1].image[n - 1], "ghcr.io/acme/app:1.0") == 0 && + probes[1].keys[n - 1] && + strcmp(probes[1].keys[n - 1], "[\"image.repository\",\"image.tag\"]") == 0; + cbm_arena_destroy(&a); + if (doc) { + yyjson_mut_doc_free(doc); + } + ASSERT_TRUE(ok); + ASSERT_EQ(n, LATER); + ASSERT_TRUE(last); + PASS(); +} + +/* 5,000 includes of one name in a loop -- a template file of the chart with + * no change, or a name that is none -- get one answer each time (asked + * once): the first spoils nothing, the second every image after it. */ +TEST(helm_includes_of_one_name_in_a_loop) { + cbm_helm_defines_t *d = helm_chart_files(); + ASSERT_NOT_NULL(d); + static const char clean[] = + "{{- range 5000 }}{{ $_ := include (print $.Template.BasePath \"/secret.yaml\") $ }}" + "{{ end }}\n" HELM_IMAGE_LINE; + static const char missing[] = + "{{- range 5000 }}{{ $_ := include (print $.Template.BasePath \"/missing.yaml\") $ }}" + "{{ end }}\n" HELM_IMAGE_LINE; + char *kept = helm_capture_chart(HELM_IMAGE_VALUES, clean, d, &HELM_APP, 2); + char *spoiled = helm_capture_chart(HELM_IMAGE_VALUES, missing, d, &HELM_APP, 2); + cbm_helm_defines_free(d); + ASSERT_STR_EQ(kept, "ghcr.io/acme/app:1.0"); + ASSERT_STR_EQ(spoiled, ""); + free(kept); + free(spoiled); + PASS(); +} + /* A compose project's .env feeds its compose file (variables in image, * build): changing it is no closure repair -- the unchanged compose file's * services would keep the old image. It rebuilds; no stale image remains. */ @@ -24811,6 +24981,13 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(helm_subchart_base_path_names_its_own_file); RUN_TEST(helm_include_of_a_missing_template_file); RUN_TEST(helm_template_file_including_itself); + RUN_TEST(helm_deep_merge_over_a_change_below); + RUN_TEST(helm_and_or_with_an_unknown_operand); + RUN_TEST(helm_later_condition_of_a_skipped_if); + RUN_TEST(helm_sort_alpha_changes_its_list); + RUN_TEST(helm_parse_error_spoils); + RUN_TEST(helm_captures_after_a_huge_capture); + RUN_TEST(helm_includes_of_one_name_in_a_loop); RUN_TEST(incremental_compose_dotenv_change_rebuilds); RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges); RUN_TEST(incremental_second_build_declarer_keeps_the_edge); From c52ca5d34c76b4f731090ab17a2dc3390dea6467 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 19 Oct 2026 23:52:05 +0200 Subject: [PATCH 19/19] feat(hooks): name the graph follow-up tools in every search augmentation When an agent greps for a symbol that the graph knows, the PreToolUse augmentation (Grep, Glob and Bash searches) lists the matching symbols. It now closes with one line that is cbm's own statement, outside the untrusted symbol block, naming the graph tools for the usual next question: [codebase-memory] For these symbols, trace_path lists callers and callees, get_code_snippet returns the exact source, and search_graph finds related symbols by name. The line carries no repository-derived value and is stated as facts, as Claude Code asks of hook context. It is appended at the string's real end into reserved room: the list loop's offset can run past its cap when a row is truncated, so appending at that offset could have written past the buffer. Searches without a hit, errors and non-search events are unchanged. Refs #509 Signed-off-by: Martin Vogel --- src/cli/hook_augment.c | 13 +++++++++++- tests/test_cli.c | 45 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/cli/hook_augment.c b/src/cli/hook_augment.c index 79a197716..ef6b7a3fe 100644 --- a/src/cli/hook_augment.c +++ b/src/cli/hook_augment.c @@ -377,6 +377,13 @@ static char *ha_build_args(const char *project, const char *token) { return out; /* caller frees */ } +/* Closes every search augmentation that found symbols: cbm's own statement, + * outside the untrusted data block, naming the graph tool for the likely next + * question at the moment the agent greps. Facts, not commands. */ +static const char HA_SEARCH_FOLLOW_UP[] = + "\n[codebase-memory] For these symbols, trace_path lists callers and callees, " + "get_code_snippet returns the exact source, and search_graph finds related symbols by name."; + /* Parse the MCP envelope returned by cbm_mcp_handle_tool and, if it is a * successful search_graph result with >=1 hit, format a compact * additionalContext string. Returns malloc'd text or NULL. @@ -432,7 +439,7 @@ static char *ha_format_context(const char *envelope, const char *token, bool *is return NULL; /* valid project, just no matching symbols */ } - char *text = malloc(4096); + char *text = malloc(4096 + sizeof(HA_SEARCH_FOLLOW_UP)); if (!text) { yyjson_doc_free(idoc); yyjson_doc_free(edoc); @@ -476,6 +483,10 @@ static char *ha_format_context(const char *envelope, const char *token, bool *is safe_path, safe_label[0] ? " " : "", safe_label); } } + /* off can run past the list cap when a row is truncated, so append at the + * real end; the allocation reserves room for the whole follow-up line. */ + size_t used = strlen(text); + snprintf(text + used, 4096 + sizeof(HA_SEARCH_FOLLOW_UP) - used, "%s", HA_SEARCH_FOLLOW_UP); yyjson_doc_free(idoc); yyjson_doc_free(edoc); diff --git a/tests/test_cli.c b/tests/test_cli.c index c02091d78..539b0df18 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -13498,6 +13498,50 @@ TEST(cli_hook_augment_context_tracks_search_json_shape) { PASS(); } +/* A search that found symbols closes with cbm's own follow-up line, after the + * untrusted symbol list and on its own line, naming the graph tools for the + * next question; it carries no repository-derived value. */ +TEST(cli_hook_augment_context_names_graph_follow_ups) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "hookproj"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/hookproj"); + cbm_node_t n = {.project = proj, + .label = "Function", + .name = "someIndexedSymbol", + .qualified_name = "hookproj.mod.someIndexedSymbol", + .file_path = "mod.py", + .start_line = 1, + .end_line = 4}; + ASSERT_GT(cbm_store_upsert_node(st, &n), 0); + char *envelope = + cbm_mcp_handle_tool(srv, "search_graph", + "{\"project\":\"hookproj\",\"name_pattern\":\".*someIndexedSymbol.*\"," + "\"limit\":5,\"format\":\"json\"}"); + ASSERT_NOT_NULL(envelope); + + bool is_error = true; + char *ctx = + cbm_hook_augment_format_context_for_testing(envelope, "someIndexedSymbol", &is_error); + ASSERT_FALSE(is_error); + ASSERT_NOT_NULL(ctx); + const char *list = strstr(ctx, "\n- hookproj.mod.someIndexedSymbol"); + const char *follow = strstr(ctx, "\n[codebase-memory] For these symbols, trace_path lists " + "callers and callees, get_code_snippet returns the exact " + "source, and search_graph finds related symbols by name."); + ASSERT_NOT_NULL(list); + ASSERT_NOT_NULL(follow); + ASSERT_TRUE(follow > list); + ASSERT_NULL(strstr(follow, "someIndexedSymbol")); + ASSERT_NULL(strstr(follow, "mod.py")); + free(ctx); + free(envelope); + cbm_mcp_server_free(srv); + PASS(); +} + /* Exercise the real PreToolUse + Bash route instead of calling only the * tokenizer seam. This pins the event guard and the graph lookup path together. */ @@ -18479,6 +18523,7 @@ SUITE(cli) { RUN_TEST(cli_codex_preflight_reports_heading_and_reason); #endif RUN_TEST(cli_hook_augment_context_tracks_search_json_shape); + RUN_TEST(cli_hook_augment_context_names_graph_follow_ups); RUN_TEST(cli_hook_augment_bash_pretooluse_reaches_augmenter); RUN_TEST(cli_hook_augment_bash_pattern_extractor); RUN_TEST(cli_hook_augment_coverage_requests_machine_json);