mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
feat(shell): shell and Make deploy commands DEPLOYS what they apply
A script or Make recipe running kubectl apply, helm upgrade or a kustomize
build piped into kubectl puts manifests, a kustomization or a chart on a
cluster, and none of it was an edge: "which script deploys this manifest"
had no answer in the graph.
A new pass (the Bash + Make layer, run at the end of the K8s pass) reads
scripts and Makefiles that name a deploy tool and links, to existing nodes
only:
- kubectl apply|create|replace -f: the File of a manifest, a directory's
manifests (-R: below it too), a glob's; -k: the Kustomization;
- kustomize build D (or kubectl kustomize D) | ... | kubectl apply -f -:
the Kustomization of D;
- helm install|upgrade [NAME] CHART, helm template piped into kubectl:
the Chart of a chart dir, else the shared external Chart of repo/name or
oci://.../name;
- find D -name G -exec kubectl apply -f {} \;: D's matching manifests.
The edge starts at the bash function or Make rule running the command,
top-level code at its file, with via (kubectl / helm / kustomize), the
deployed path or chart, and the line.
Words are evaluated by constant propagation, nothing is executed:
assignments (local / export / readonly / declare, arrays), ${X:-default},
$0 / BASH_SOURCE, $(dirname ..), $(cd .. && pwd), $(realpath ..),
$(git rev-parse --show-toplevel); cd / pushd / popd track the directory.
Anything else is unknown, and an unknown target is no edge: a value set in
a branch or loop is unknown after it, a subshell's changes do not leak, an
unquoted unknown word that may shift helm's positionals blocks the chart.
With no directory tracked, a relative path resolves against the root and
the script's directory, linked only when exactly one of them has a target.
Make: the Makefile's variables expand into each recipe line ($(shell pwd) /
$(CURDIR) = the Makefile's directory; a conditional's value is unknown),
which is then read as a shell line.
Measured on 17 repositories against an independent reading of the scripts
(its own tokenizer, variable and directory tracking, the file system): 86
of 86 deploys linked, 86 of 86 correct (k8s.io 66, kubermatic 12, gardener
8). Only DEPLOYS edges and 3 external Charts were added; three runs build
identical graphs; index time unchanged within 1 s (deckhouse 10 s, gardener
5-6 s, k8s.io 3-4 s, median of 3). Nine incremental edits (deploy lines
edited, removed and added, a Makefile, a deployed manifest, a new manifest
and a new non-manifest in an applied directory) leave the graph a fresh
index builds.
tree-sitter-bash folds `a | b` followed by a line `cat <<EOF | c` into
`a | (b cat | c)`; that shape continues the outer pipe.
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
1 parent
b3a5b788f7
commit
3323cae0f8
7 files changed
+2886
-1
No files matched your search
@@ -519,6 +519,7 @@ PIPELINE_SRCS = \
|
||||
src/pipeline/pass_infrascan.c \
|
||||
src/pipeline/pass_k8s.c \
|
||||
src/pipeline/pass_operator.c \
|
||||
src/pipeline/pass_shell.c \
|
||||
src/pipeline/pass_similarity.c \
|
||||
src/pipeline/pass_semantic_edges.c \
|
||||
src/pipeline/pass_complexity.c \
|
||||
|
||||
@@ -74,10 +74,15 @@ REFS=(
|
||||
"kustomize|https://github.com/kubernetes-sigs/kustomize|api/krusty,api/internal/target"
|
||||
"flux|https://github.com/fluxcd/pkg|kustomize"
|
||||
"kube|https://github.com/kubernetes-sigs/controller-tools|pkg"
|
||||
# The Bash (+ Make) layer (deploy commands, constant propagation) was written
|
||||
# from the bash, GNU make, kubectl and helm documentation. Scanned against
|
||||
# bash-language-server (MIT), the leading OSS bash language server.
|
||||
"bash|https://github.com/bash-lsp/bash-language-server|server/src"
|
||||
)
|
||||
|
||||
# Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too).
|
||||
INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c)
|
||||
INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c
|
||||
src/pipeline/pass_shell.c)
|
||||
|
||||
ONLY_LANG=""; REFRESH=0; LIST_ONLY=0
|
||||
while [ $# -gt 0 ]; do
|
||||
|
||||
@@ -3412,6 +3412,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
k8s_link_includes(ctx, &bundles, files, file_count);
|
||||
k8s_bundles_free(&bundles);
|
||||
k8s_kusts_free(&kusts);
|
||||
/* Shell and Make deploy commands, against the nodes linked above. */
|
||||
cbm_pipeline_pass_shell(ctx, files, file_count);
|
||||
|
||||
cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count));
|
||||
(void)helm_count;
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1042,6 +1042,10 @@ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx);
|
||||
* Runs after USAGE resolution. */
|
||||
int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx);
|
||||
|
||||
/* Shell scripts and Make recipes (pass_shell.c): the commands that put
|
||||
* manifests, kustomizations or charts on a cluster DEPLOYS them. Run at the
|
||||
* end of the K8s pass, over the same files. */
|
||||
void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count);
|
||||
/* Pre-dump pass: markdown → file REFERENCES_FILE linking. */
|
||||
int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx);
|
||||
|
||||
@@ -1124,6 +1128,7 @@ extern _Atomic uint64_t g_importance_store_rows;
|
||||
* examines (pass_operator.c). Independent copies must grow them linearly. */
|
||||
extern _Atomic uint64_t g_k8s_candidates;
|
||||
extern _Atomic uint64_t g_op_candidates;
|
||||
extern _Atomic uint64_t g_sh_candidates;
|
||||
|
||||
/* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */
|
||||
|
||||
|
||||
@@ -1696,6 +1696,210 @@ TEST(probe_gitops_objects_deploy_their_units) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Every DEPLOYS edge as "source -> target (via)", sorted, one per line: a
|
||||
* function by name, top-level code as "@<file>"; a File or Kustomization by
|
||||
* its path, a Chart by name ("(ext)": the external one). NULL on failure;
|
||||
* the caller frees. */
|
||||
static char *gpf_deploys(const GpfFile *files, int nfiles) {
|
||||
GpfProj lp;
|
||||
cbm_store_t *store = gpf_index_files(&lp, files, nfiles);
|
||||
char *out = NULL;
|
||||
if (store) {
|
||||
sqlite3_stmt *st = NULL;
|
||||
const char *sql =
|
||||
"SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT "
|
||||
"CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END "
|
||||
"|| ' -> ' || CASE WHEN t.label='Chart' THEN 'chart:' || t.name || "
|
||||
"CASE WHEN instr(t.qualified_name,'__helm_dep__') THEN '(ext)' ELSE '' END "
|
||||
"ELSE t.label || ':' || t.file_path END "
|
||||
"|| ' (' || json_extract(e.properties,'$.via') || ')' AS x "
|
||||
"FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id "
|
||||
"WHERE e.project=?1 AND e.type='DEPLOYS' ORDER BY x)";
|
||||
if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) {
|
||||
sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC);
|
||||
if (sqlite3_step(st) == SQLITE_ROW) {
|
||||
out = strdup((const char *)sqlite3_column_text(st, 0));
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(st);
|
||||
}
|
||||
gpf_cleanup(&lp, store);
|
||||
return out;
|
||||
}
|
||||
|
||||
#define GPF_ASSERT_DEPLOYS(files, n, want) \
|
||||
do { \
|
||||
char *got_ = gpf_deploys(files, n); \
|
||||
if (!got_ || strcmp(got_, want) != 0) { \
|
||||
fprintf(stderr, "DEPLOYS got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \
|
||||
free(got_); \
|
||||
FAIL("DEPLOYS edge set"); \
|
||||
} \
|
||||
free(got_); \
|
||||
} while (0)
|
||||
|
||||
#define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n"
|
||||
#define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"
|
||||
|
||||
/* kubectl apply -f deploys a manifest, a directory's manifests (-R: below
|
||||
* it too; not a file holding no objects), a glob's; -k and a kustomize build
|
||||
* piped into kubectl a Kustomization. The source is the function running the
|
||||
* command, top-level code its file; paths resolve against the cd'd root. */
|
||||
TEST(probe_shell_kubectl_deploys_what_it_applies) {
|
||||
static const GpfFile files[] = {
|
||||
{"manifests/app.yaml", GPF_CM("app")},
|
||||
{"manifests/svc.yaml", GPF_CM("svc")},
|
||||
{"manifests/notes.yaml", "notes:\n - not an object\n"},
|
||||
{"manifests/nested/cm.yaml", GPF_CM("cm")},
|
||||
{"crds/a.yaml", GPF_CM("a")},
|
||||
{"crds/b.yaml", GPF_CM("b")},
|
||||
{"overlays/prod/kustomization.yaml", GPF_KUST},
|
||||
{"overlays/dev/kustomization.yaml", GPF_KUST},
|
||||
{"hack/deploy.sh", "#!/usr/bin/env bash\n"
|
||||
"cd \"$(dirname \"$0\")/..\"\n\n"
|
||||
"apply_all() {\n"
|
||||
" kubectl apply -f manifests/app.yaml\n"
|
||||
" kubectl --namespace \"$NS\" apply -f manifests\n"
|
||||
" kubectl apply -Rf manifests/nested\n"
|
||||
"}\n\n"
|
||||
"kubectl apply -f 'crds/*.yaml'\n"
|
||||
"kubectl apply -k overlays/prod\n"
|
||||
"kustomize build overlays/dev | sed s/a/b/ | kubectl apply -f -\n"
|
||||
"cat <<EOF | kubectl apply -f -\nkind: Inline\nEOF\n"}};
|
||||
GPF_ASSERT_DEPLOYS(files, 9,
|
||||
"@hack/deploy.sh -> File:crds/a.yaml (kubectl)\n"
|
||||
"@hack/deploy.sh -> File:crds/b.yaml (kubectl)\n"
|
||||
"@hack/deploy.sh -> Resource:overlays/dev/kustomization.yaml (kustomize)\n"
|
||||
"@hack/deploy.sh -> Resource:overlays/prod/kustomization.yaml (kubectl)\n"
|
||||
"apply_all -> File:manifests/app.yaml (kubectl)\n"
|
||||
"apply_all -> File:manifests/nested/cm.yaml (kubectl)\n"
|
||||
"apply_all -> File:manifests/svc.yaml (kubectl)");
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* helm install / upgrade deploys its chart: a chart dir (through options,
|
||||
* a wrapper, $(git rev-parse --show-toplevel)), else the external chart of
|
||||
* repo/name or oci://.../name; a helm template piped into kubectl too. No
|
||||
* edge from an unknown word that may shift the chart's position, nor from
|
||||
* a variable an if/else leaves undecided. */
|
||||
TEST(probe_shell_helm_deploys_its_chart) {
|
||||
static const GpfFile files[] = {
|
||||
{"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"},
|
||||
{"charts/web/values-prod.yaml", "replicas: 2\n"},
|
||||
{"charts/api/Chart.yaml", "apiVersion: v2\nname: api\nversion: 0.1.0\n"},
|
||||
{"scripts/install.sh",
|
||||
"#!/bin/sh\n"
|
||||
"ROOT=\"$(git rev-parse --show-toplevel)\"\n"
|
||||
"NS=${NAMESPACE:-web}\n"
|
||||
"install_web() {\n"
|
||||
" retry 3 helm upgrade --install --namespace \"$NS\" --create-namespace --atomic "
|
||||
"web \"$ROOT/charts/web/\"\n"
|
||||
"}\n"
|
||||
"install_cache() {\n"
|
||||
" helm install cache bitnami/redis --version 1.2.3\n"
|
||||
" helm install queue oci://registry.example.com/charts/nats\n"
|
||||
"}\n"
|
||||
"render_web() {\n"
|
||||
" helm template web charts/web -f charts/web/values-prod.yaml | kubectl apply -f -\n"
|
||||
"}\n"
|
||||
"split_args() {\n"
|
||||
" helm upgrade $ARGS charts/api\n"
|
||||
"}\n"
|
||||
"branchy() {\n"
|
||||
" CHART=charts/web\n"
|
||||
" if [ -n \"$PROD\" ]; then CHART=charts/api; fi\n"
|
||||
" helm upgrade --install x \"$CHART\"\n"
|
||||
"}\n"}};
|
||||
GPF_ASSERT_DEPLOYS(files, 4,
|
||||
"install_cache -> chart:nats(ext) (helm)\n"
|
||||
"install_cache -> chart:redis(ext) (helm)\n"
|
||||
"install_web -> chart:web (helm)\n"
|
||||
"render_web -> chart:web (helm)");
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Constant propagation: SCRIPT_ROOT from $(cd "$(dirname ...)" && pwd -P),
|
||||
* pushd / popd, an option whose value is unknown, an array holding the
|
||||
* command; a subshell's cd does not leak; with no directory tracked a path
|
||||
* resolves against the script's dir or the root -- only when one of the two
|
||||
* has it. */
|
||||
TEST(probe_shell_paths_follow_the_script) {
|
||||
static const GpfFile files[] = {
|
||||
{"apps/web/deployment.yaml", GPF_CM("deployment")},
|
||||
{"apps/web/service.yaml", GPF_CM("service")},
|
||||
{"apps/web/sub/cm.yaml", GPF_CM("cm")},
|
||||
{"apps/web/sub/extra.yaml", GPF_CM("extra")},
|
||||
{"apps/web/config.yaml", GPF_CM("config-here")},
|
||||
{"config.yaml", GPF_CM("config-root")},
|
||||
{"apps/web/deploy.sh",
|
||||
"#!/usr/bin/env bash\n"
|
||||
"SCRIPT_ROOT=$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd -P)\n"
|
||||
"context=\"${KUBECTL_CONTEXT:-gke}\"\n"
|
||||
"if ! kubectl config get-contexts \"${context}\"; then\n"
|
||||
" context=\"other\"\n"
|
||||
"fi\n"
|
||||
"pushd \"${SCRIPT_ROOT}\" >/dev/null\n"
|
||||
"kubectl --context=\"${context}\" apply -f .\n"
|
||||
"popd >/dev/null\n"
|
||||
"( cd sub && kubectl apply -f cm.yaml )\n"
|
||||
"kubectl apply -f extra.yaml\n"
|
||||
"ambiguous() { kubectl apply -f config.yaml; }\n"
|
||||
"unique() { kubectl apply -f deployment.yaml; }\n"
|
||||
"deploy() {\n"
|
||||
" local kubectl=(\n"
|
||||
" kubectl\n"
|
||||
" --context=\"${context}\"\n"
|
||||
" )\n"
|
||||
" \"${kubectl[@]}\" apply -f service.yaml\n"
|
||||
"}\n"}};
|
||||
GPF_ASSERT_DEPLOYS(files, 7,
|
||||
"@apps/web/deploy.sh -> File:apps/web/config.yaml (kubectl)\n"
|
||||
"@apps/web/deploy.sh -> File:apps/web/deployment.yaml (kubectl)\n"
|
||||
"@apps/web/deploy.sh -> File:apps/web/service.yaml (kubectl)\n"
|
||||
"@apps/web/deploy.sh -> File:apps/web/sub/cm.yaml (kubectl)\n"
|
||||
"deploy -> File:apps/web/service.yaml (kubectl)\n"
|
||||
"unique -> File:apps/web/deployment.yaml (kubectl)");
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A Make recipe runs in its Makefile's directory with the Makefile's
|
||||
* variables expanded ($(shell pwd), ?=, a conditional's: unknown): a
|
||||
* kustomize build into kubectl, kubectl -f a directory, find -name -exec
|
||||
* kubectl apply -f {}. */
|
||||
TEST(probe_make_recipes_deploy) {
|
||||
static const GpfFile files[] = {
|
||||
{"operator/config/default/kustomization.yaml", GPF_KUST},
|
||||
{"operator/config/prod/kustomization.yaml", GPF_KUST},
|
||||
{"operator/config/dev/kustomization.yaml", GPF_KUST},
|
||||
{"operator/config/crd/widgets.yaml", GPF_CM("widgets")},
|
||||
{"operator/config/flux/flux-a.yaml", GPF_CM("flux-a")},
|
||||
{"operator/config/flux/other.yaml", GPF_CM("other")},
|
||||
{"operator/config/flux/sub/flux-b.yaml", GPF_CM("flux-b")},
|
||||
{"operator/Makefile", "LOCALBIN ?= $(shell pwd)/bin\n"
|
||||
"KUBECTL ?= kubectl\n"
|
||||
"KUSTOMIZE ?= $(LOCALBIN)/kustomize\n"
|
||||
"ifeq ($(ENV),prod)\n"
|
||||
"OVERLAY = config/prod\n"
|
||||
"else\n"
|
||||
"OVERLAY = config/dev\n"
|
||||
"endif\n\n"
|
||||
"deploy:\n"
|
||||
"\t$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -\n\n"
|
||||
"install:\n"
|
||||
"\t@$(KUBECTL) apply -f config/crd\n\n"
|
||||
"flux:\n"
|
||||
"\tfind ./config/flux -type f -name \"flux-*\" -exec $(KUBECTL) "
|
||||
"apply -f {} \\;\n\n"
|
||||
"overlay:\n"
|
||||
"\t$(KUBECTL) apply -k $(OVERLAY)\n"}};
|
||||
GPF_ASSERT_DEPLOYS(files, 8,
|
||||
"deploy -> Resource:operator/config/default/kustomization.yaml (kustomize)\n"
|
||||
"flux -> File:operator/config/flux/flux-a.yaml (kubectl)\n"
|
||||
"flux -> File:operator/config/flux/sub/flux-b.yaml (kubectl)\n"
|
||||
"install -> File:operator/config/crd/widgets.yaml (kubectl)");
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Helm symbolic (Phase 2) ────────────────────────────────────────── */
|
||||
|
||||
/* A chart template is YAML only once rendered: a document opened by
|
||||
@@ -3367,6 +3571,10 @@ SUITE(grammar_probe_f) {
|
||||
RUN_TEST(probe_mcp_search_filters_kind_and_namespace);
|
||||
RUN_TEST(probe_mcp_trace_topology);
|
||||
RUN_TEST(probe_gitops_objects_deploy_their_units);
|
||||
RUN_TEST(probe_shell_kubectl_deploys_what_it_applies);
|
||||
RUN_TEST(probe_shell_helm_deploys_its_chart);
|
||||
RUN_TEST(probe_shell_paths_follow_the_script);
|
||||
RUN_TEST(probe_make_recipes_deploy);
|
||||
RUN_TEST(probe_helm_templated_objects_are_resources);
|
||||
RUN_TEST(probe_helm_include_resolves_in_the_chart_family);
|
||||
RUN_TEST(probe_k8s_selector_stays_in_its_directory);
|
||||
|
||||
@@ -3422,6 +3422,41 @@ TEST(incremental_k8s_edits_reach_other_files) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A script's deploy commands link nodes of other files (pass_shell.c): the
|
||||
* closure repair re-reads the changed script alone, so editing it must still
|
||||
* leave the edges a fresh index builds. */
|
||||
TEST(incremental_shell_deploy_edit_keeps_its_edges) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_shinc_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmp)) {
|
||||
FAIL("tmpdir");
|
||||
}
|
||||
write_temp_file(tmp, "manifests/app.yaml",
|
||||
"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: app\n");
|
||||
write_temp_file(tmp, "hack/deploy.sh",
|
||||
"#!/bin/sh\ncd \"$(dirname \"$0\")/..\"\nkubectl apply -f manifests/app.yaml\n");
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/shinc.db", tmp);
|
||||
static const char deploys[] = "SELECT COUNT(*) FROM edges WHERE type='DEPLOYS'";
|
||||
|
||||
int rc_full = k8s_inc_index(tmp, db_path);
|
||||
int deploys_full = k8s_inc_count(db_path, deploys);
|
||||
write_temp_file(tmp, "hack/deploy.sh",
|
||||
"#!/bin/sh\ncd \"$(dirname \"$0\")/..\"\nkubectl apply -f manifests/app.yaml\n"
|
||||
"# edited\n");
|
||||
int rc_edit = k8s_inc_index(tmp, db_path);
|
||||
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
|
||||
int deploys_after = k8s_inc_count(db_path, deploys);
|
||||
th_rmtree(tmp);
|
||||
|
||||
ASSERT_EQ(rc_full, 0);
|
||||
ASSERT_EQ(deploys_full, 1);
|
||||
ASSERT_EQ(rc_edit, 0);
|
||||
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
|
||||
ASSERT_EQ(deploys_after, 1);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage,
|
||||
* while the shared name registry must NOT leak those relations into other
|
||||
* languages' textual resolution: a Python call or identifier sharing the
|
||||
@@ -19046,6 +19081,7 @@ SUITE(pipeline_semantic_manifest_repro) {
|
||||
RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge);
|
||||
RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419);
|
||||
RUN_TEST(incremental_k8s_edits_reach_other_files);
|
||||
RUN_TEST(incremental_shell_deploy_edit_keeps_its_edges);
|
||||
RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
|
||||
RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
|
||||
RUN_TEST(pipeline_dbt_jinja_lineage);
|
||||
|
||||
Reference in new issue
Block a user