feat(docker): a Dockerfile DEPENDS_ON the images it builds on

Dockerfiles were Module + ENV/ARG Variables only: the 261 FROM and
114 COPY --from lines of the infra corpora joined nothing, and build
files named for a variant were not indexed at all.

- pass_docker (new, end of the K8s pass): FROM <image> and
  COPY --from=<image> -> DEPENDS_ON {via: from | copy-from, image,
  stage, line} from the Dockerfile's Module to an Image node. ARG
  defaults are expanded (global ARGs in FROM, a stage's ARG / ENV in
  COPY); a stage (by name or index), scratch and an ARG without a value
  are no image.
- Image (new label, as approved): one node per repository, normalised
  (docker.io for no registry host, library/ for an official image, a
  registry host and port kept); tag and digest stay on the edge.
- Detection: Dockerfile.<x>, Dockerfile-<x>, Dockerfile_<x>,
  Containerfile[.<x>] and <x>.Dockerfile are Dockerfiles, unless the
  suffix is a language extension of its own (Dockerfile.j2 stays a
  Jinja template) or a BuildKit ignore file.
- QNs: a Dockerfile.<variant> keeps its suffix (fqn.c and helpers.c,
  which must agree); dropping it gave the variant the plain
  Dockerfile's QN, merging both Modules. Covered by this branch's
  index-format bump.
- Incremental: an Image no DEPENDS_ON / USAGE edge touches any more is
  dropped on both routes, like a Process without SPAWNS; the
  unreferenced-node GC takes a list of edge types.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-10-11 11:42:35 +02:00
1 parent a48852d77e
commit 2ec0185b9a
17 files changed
+760 -31

No files matched your search

+1
View File
@@ -521,6 +521,7 @@ PIPELINE_SRCS = \
src/pipeline/pass_operator.c \
src/pipeline/pass_shell.c \
src/pipeline/pass_terraform.c \
src/pipeline/pass_docker.c \
src/pipeline/pass_ci.c \
src/pipeline/pass_similarity.c \
src/pipeline/pass_semantic_edges.c \
+22
View File
@@ -1651,7 +1651,29 @@ bool cbm_is_module_level(TSNode node, CBMLanguage lang) {
// Mirrors Go's fqn.Compute(): project + path_parts_dotted + name
// Internal helper: find extension start in basename (returns length without ext)
/* A container build file named for its variant ("Dockerfile.debug") keeps
* the suffix (no language extension; dropping it gives the variant the QN of
* the plain Dockerfile beside it). Mirrors strip_file_extension in
* src/pipeline/fqn.c -- both must agree. */
static bool build_file_variant(const char *s, size_t len) {
size_t b = len;
while (b > 0 && s[b - SKIP_ONE] != '/') {
b--;
}
static const char *const BASES[] = {"Dockerfile.", "Containerfile."};
for (size_t i = 0; i < sizeof(BASES) / sizeof(BASES[0]); i++) {
size_t bl = strlen(BASES[i]);
if (len - b > bl && strncmp(s + b, BASES[i], bl) == 0) {
return true;
}
}
return false;
}
static size_t strip_ext_len(const char *s, size_t len) {
if (build_file_variant(s, len)) {
return len;
}
for (size_t i = len; i > 0; i--) {
if (s[i - SKIP_ONE] == '.') {
/* A dot at the very start of a filename segment (index 0, or right
+1 -1
View File
@@ -83,7 +83,7 @@ REFS=(
# Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too).
INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c
src/pipeline/pass_shell.c src/pipeline/pass_terraform.c
src/pipeline/pass_ci.c)
src/pipeline/pass_ci.c src/pipeline/pass_docker.c)
ONLY_LANG=""; REFRESH=0; LIST_ONLY=0
while [ $# -gt 0 ]; do
+31
View File
@@ -93,6 +93,7 @@ static const ext_entry_t EXT_TABLE[] = {
/* Dockerfile */
{".dockerfile", CBM_LANG_DOCKERFILE},
{".Dockerfile", CBM_LANG_DOCKERFILE},
/* Elixir */
{".ex", CBM_LANG_ELIXIR},
@@ -713,6 +714,7 @@ typedef struct {
static const filename_entry_t FILENAME_TABLE[] = {
{"CMakeLists.txt", CBM_LANG_CMAKE},
{"Containerfile", CBM_LANG_DOCKERFILE},
{"Dockerfile", CBM_LANG_DOCKERFILE},
{"GNUmakefile", CBM_LANG_MAKEFILE},
{"Makefile", CBM_LANG_MAKEFILE},
@@ -957,6 +959,32 @@ CBMLanguage cbm_language_for_extension(const char *ext) {
return lang_for_extension(cbm_get_user_lang_config(), ext);
}
/* A container build file named for its variant: "Dockerfile.dev",
* "Dockerfile-test", "Containerfile.arm64". Not when the suffix is a language
* extension of its own (Dockerfile.j2 is a Jinja template) nor a BuildKit
* ignore file (Dockerfile.dockerignore). */
static bool lang_dockerfile_variant(const cbm_userconfig_t *ucfg, const char *filename) {
static const char *const BASES[] = {"Dockerfile", "Containerfile"};
static const char IGNORE[] = ".dockerignore";
for (size_t i = 0; i < sizeof(BASES) / sizeof(BASES[0]); i++) {
size_t bl = strlen(BASES[i]);
char sep = filename[bl];
if (strncmp(filename, BASES[i], bl) != 0 || (sep != '.' && sep != '-' && sep != '_') ||
!filename[bl + SKIP_ONE]) {
continue;
}
size_t fl = strlen(filename);
if (fl >= sizeof(IGNORE) - SKIP_ONE &&
strcmp(filename + fl - (sizeof(IGNORE) - SKIP_ONE), IGNORE) == 0) {
return false;
}
const char *last_dot = strrchr(filename, '.');
return !last_dot || last_dot < filename + bl ||
lang_for_extension(ucfg, last_dot) == CBM_LANG_COUNT;
}
return false;
}
static CBMLanguage lang_for_filename(const cbm_userconfig_t *ucfg, const char *filename) {
if (!filename || !filename[0]) {
return CBM_LANG_COUNT;
@@ -977,6 +1005,9 @@ static CBMLanguage lang_for_filename(const cbm_userconfig_t *ucfg, const char *f
if (strncmp(filename, ".env.", SLEN(".env.")) == 0) {
return CBM_LANG_DOTENV;
}
if (lang_dockerfile_variant(ucfg, filename)) {
return CBM_LANG_DOCKERFILE;
}
/* Fall back to extension-based lookup.
* For compound extensions (e.g. ".blade.php") defined in the user config,
+18 -7
View File
@@ -1498,8 +1498,23 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) {
return deleted_count;
}
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *edge_type) {
if (!gb || !label || !edge_type) {
/* Whether an edge of `types` (NULL-terminated) touches node `id`. */
static bool gbuf_node_touched(const cbm_gbuf_t *gb, int64_t id, const char *const *types) {
for (int t = 0; types[t]; t++) {
const cbm_gbuf_edge_t **edges = NULL;
int count = 0;
if ((cbm_gbuf_find_edges_by_target_type(gb, id, types[t], &edges, &count) == 0 &&
count > 0) ||
(cbm_gbuf_find_edges_by_source_type(gb, id, types[t], &edges, &count) == 0 &&
count > 0)) {
return true;
}
}
return false;
}
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) {
if (!gb || !label || !edge_types || !edge_types[0]) {
return CBM_NOT_FOUND;
}
node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label);
@@ -1514,13 +1529,9 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *
}
int n = 0;
for (int i = 0; i < arr->count; i++) {
const cbm_gbuf_edge_t **edges = NULL;
int count = 0;
const char *qn = arr->items[i]->qualified_name;
cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL;
if (node &&
cbm_gbuf_find_edges_by_target_type(gb, node->id, edge_type, &edges, &count) == 0 &&
count == 0) {
if (node && !gbuf_node_touched(gb, node->id, edge_types)) {
victims[n++] = node;
}
}
+6 -5
View File
@@ -110,11 +110,12 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label);
* Used by incremental indexing to remove stale nodes before re-extraction. */
int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path);
/* Delete every `label` node that no `edge_type` edge targets (with any edges
* it still has); returns how many. For nodes that exist only as such a
* target and carry no file (Process): incremental indexing drops them when
* their last referrer is gone, as a full index never makes them. */
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *edge_type);
/* Delete every `label` node that no edge of `edge_types` (NULL-terminated)
* touches, as source or target (with any edges it still has); returns how
* many. For nodes that exist only through such edges and carry no file
* (Process, Image): incremental indexing drops them when their last referrer
* is gone, as a full index never makes them. */
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types);
/* Bulk-load all nodes and edges for a project from an existing SQLite DB
* into this graph buffer. Returns 0 on success. */
+12
View File
@@ -63,9 +63,21 @@ static char *join_segments(const char **segments, int count) {
}
/* Strip file extension from the last path component. */
/* A container build file named for its variant ("Dockerfile.debug") keeps
* the suffix: it is no language extension, and dropping it would give the
* variant the QN of the plain Dockerfile beside it. Mirrors strip_ext_len in
* internal/cbm/helpers.c -- both must agree. */
static bool fqn_build_file_variant(const char *base) {
return strncmp(base, "Dockerfile.", strlen("Dockerfile.")) == 0 ||
strncmp(base, "Containerfile.", strlen("Containerfile.")) == 0;
}
static void strip_file_extension(char *path) {
char *last_slash = strrchr(path, '/');
char *start = last_slash ? last_slash + SKIP_ONE : path;
if (fqn_build_file_variant(start)) {
return;
}
char *ext = strrchr(start, '.');
if (ext) {
*ext = '\0';
+452
View File
@@ -0,0 +1,452 @@
/*
* pass_docker.c — the container build layer.
*
* PHASE 4 P4.1a: what a Dockerfile builds on. Each base it names DEPENDS_ON
* an Image node -- an edge from the Dockerfile's Module:
* FROM <image> [AS <stage>] {via:from}
* COPY --from=<image> ... {via:copy-from}
* A stage of the same file is no image (FROM builder, COPY --from=builder or
* --from=0), nor is `scratch`. ${ARG} references are expanded from the ARG
* defaults in scope (global ARGs before the first FROM; a stage's own ARG and
* ENV); a reference without a value leaves the image unknown, and an unknown
* image is no edge.
*
* Image identity is the normalised repository: docker.io for an image
* without a registry host, library/ for an official one-segment image;
* tag and digest stay on the edge. Run at the end of the K8s pass.
*/
#include "foundation/constants.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/limits.h"
#include "foundation/log.h"
#include "foundation/mem_core.h"
#include "foundation/str_util.h"
#include "graph_buffer/graph_buffer.h"
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "cbm.h"
#include "helpers.h"
#include "lang_specs.h"
#include <ctype.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum { DK_MAX_STAGES = 64 };
typedef struct dk_var {
const char *name;
const char *value; /* NULL: declared without a value */
struct dk_var *next;
} dk_var_t;
typedef struct {
cbm_pipeline_ctx_t *ctx;
CBMArena *a;
const char *src;
const char *file;
int64_t module;
dk_var_t *global; /* ARGs before the first FROM */
dk_var_t *local; /* the current stage's ARG / ENV */
const char *stages[DK_MAX_STAGES]; /* stage names; NULL: unnamed */
int stage; /* index of the current stage; -1 before FROM */
int edges;
} dk_t;
static bool dk_type(TSNode n, const char *t) {
return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0;
}
static const char *dk_text(const dk_t *d, TSNode n) {
uint32_t b = ts_node_start_byte(n);
uint32_t e = ts_node_end_byte(n);
char *s = cbm_arena_alloc(d->a, (size_t)(e - b) + SKIP_ONE);
if (s) {
memcpy(s, d->src + b, e - b);
s[e - b] = '\0';
}
return s;
}
/* A value as written: quotes of a quoted string removed. */
static const char *dk_value(const dk_t *d, TSNode n) {
const char *s = dk_text(d, n);
size_t l = s ? strlen(s) : 0;
if (l >= PAIR_LEN && (s[0] == '"' || s[0] == '\'') && s[l - SKIP_ONE] == s[0]) {
char *v = cbm_arena_alloc(d->a, l - SKIP_ONE);
if (v) {
memcpy(v, s + SKIP_ONE, l - PAIR_LEN);
v[l - PAIR_LEN] = '\0';
}
return v;
}
return s;
}
static const dk_var_t *dk_find(const dk_var_t *v, const char *name, size_t nl) {
for (; v; v = v->next) {
if (strlen(v->name) == nl && strncmp(v->name, name, nl) == 0) {
return v;
}
}
return NULL;
}
static void dk_set(dk_t *d, dk_var_t **list, const char *name, const char *value) {
dk_var_t *v = cbm_arena_alloc(d->a, sizeof(*v));
if (v) {
*v = (dk_var_t){.name = name, .value = value, .next = *list};
*list = v;
}
}
/* ASCII case-insensitive equality (stage names, `scratch`). */
static bool dk_ieq(const char *a, const char *b) {
for (; *a && *b; a++, b++) {
if (tolower((unsigned char)*a) != tolower((unsigned char)*b)) {
return false;
}
}
return *a == *b;
}
static bool dk_name_char(char c) {
return isalnum((unsigned char)c) || c == '_';
}
/* `text` with $X / ${X} / ${X:-d} / ${X-d} expanded from `vars`; NULL when a
* reference has no value. */
static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text) {
size_t cap = strlen(text) + SKIP_ONE;
size_t len = 0;
char *out = cbm_arena_alloc(d->a, cap);
for (const char *p = text; out && *p;) {
if (*p != '$') {
if (len + PAIR_LEN > cap) {
char *grown = cbm_arena_alloc(d->a, cap * PAIR_LEN);
if (!grown) {
return NULL;
}
memcpy(grown, out, len);
out = grown;
cap *= PAIR_LEN;
}
out[len++] = *p++;
continue;
}
const char *name = p + SKIP_ONE;
bool braced = *name == '{';
if (braced) {
name++;
}
size_t nl = 0;
while (dk_name_char(name[nl])) {
nl++;
}
if (nl == 0) {
return NULL;
}
const char *after = name + nl;
const char *dflt = NULL;
size_t dl = 0;
if (braced) {
const char *close = strchr(after, '}');
if (!close) {
return NULL;
}
if ((after[0] == ':' && after[SKIP_ONE] == '-') || after[0] == '-') {
dflt = after + (after[0] == ':' ? PAIR_LEN : SKIP_ONE);
dl = (size_t)(close - dflt);
} else if (after != close) {
return NULL; /* ${X:+y} and other forms: unknown */
}
after = close + SKIP_ONE;
}
const dk_var_t *v = dk_find(vars, name, nl);
const char *val = v && v->value ? v->value : NULL;
size_t vl = val ? strlen(val) : dl;
if (!val && !dflt) {
return NULL;
}
const char *from = val ? val : dflt;
if (len + vl + SKIP_ONE > cap) {
size_t ncap = (len + vl + SKIP_ONE) * PAIR_LEN;
char *grown = cbm_arena_alloc(d->a, ncap);
if (!grown) {
return NULL;
}
memcpy(grown, out, len);
out = grown;
cap = ncap;
}
memcpy(out + len, from, vl);
len += vl;
p = after;
}
if (out) {
out[len] = '\0';
}
return out;
}
/* The repository identity of image reference `ref` into out. */
static bool dk_normalize(const char *ref, char *out, size_t sz) {
char r[CBM_SZ_512];
snprintf(r, sizeof(r), "%s", ref);
char *at = strchr(r, '@');
if (at) {
*at = '\0';
}
char *slash = strrchr(r, '/');
char *colon = strchr(slash ? slash : r, ':');
if (colon) {
*colon = '\0';
}
if (!r[0] || strchr(r, ' ') || strchr(r, '$')) {
return false;
}
char *first_end = strchr(r, '/');
if (!first_end) {
snprintf(out, sz, "docker.io/library/%s", r);
return true;
}
size_t fl = (size_t)(first_end - r);
bool host = memchr(r, '.', fl) || memchr(r, ':', fl) ||
(fl == strlen("localhost") && strncmp(r, "localhost", fl) == 0);
if (!host) {
snprintf(out, sz, "docker.io/%s", r);
return true;
}
const char *rest = first_end + SKIP_ONE;
bool hub =
(fl == strlen("docker.io") && strncmp(r, "docker.io", fl) == 0) ||
(fl == strlen("index.docker.io") && strncmp(r, "index.docker.io", fl) == 0) ||
(fl == strlen("registry-1.docker.io") && strncmp(r, "registry-1.docker.io", fl) == 0);
if (hub) {
snprintf(out, sz, "docker.io/%s%s", strchr(rest, '/') ? "" : "library/", rest);
} else {
snprintf(out, sz, "%.*s/%s", (int)fl, r, rest);
}
return true;
}
/* An earlier stage of this file: its name, or a stage index. */
static bool dk_is_stage(const dk_t *d, const char *ref) {
bool digits = ref[0] != '\0';
for (const char *p = ref; *p; p++) {
digits = digits && isdigit((unsigned char)*p);
}
if (digits) {
return true;
}
for (int i = 0; i < d->stage && i < DK_MAX_STAGES; i++) {
if (d->stages[i] && dk_ieq(d->stages[i], ref)) {
return true;
}
}
return false;
}
/* DEPENDS_ON from the Dockerfile to the Image of resolved reference `ref`. */
static void dk_link(dk_t *d, const char *ref, const char *via, int line) {
char repo[CBM_SZ_512];
if (dk_ieq(ref, "scratch") || !dk_normalize(ref, repo, sizeof(repo))) {
return;
}
char qn[CBM_SZ_1K];
snprintf(qn, sizeof(qn), "__image__%s", repo);
char esc_repo[CBM_SZ_1K];
cbm_json_escape(esc_repo, (int)sizeof(esc_repo), repo);
char node_props[CBM_SZ_2K];
snprintf(node_props, sizeof(node_props), "{\"repository\":\"%s\"}", esc_repo);
int64_t img = cbm_gbuf_upsert_node(d->ctx->gbuf, "Image", repo, qn, "", 0, 0, node_props);
if (img <= 0) {
return;
}
char esc_ref[CBM_SZ_1K];
cbm_json_escape(esc_ref, (int)sizeof(esc_ref), ref);
char props[CBM_SZ_2K];
snprintf(props, sizeof(props), "{\"via\":\"%s\",\"image\":\"%s\",\"stage\":%d,\"line\":%d}",
via, esc_ref, d->stage, line);
cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, img, "DEPENDS_ON", props);
d->edges++;
}
static int dk_line(TSNode n) {
return (int)ts_node_start_point(n).row + SKIP_ONE;
}
static void dk_arg(dk_t *d, TSNode ins) {
TSNode nm = ts_node_child_by_field_name(ins, "name", (uint32_t)strlen("name"));
TSNode df = ts_node_child_by_field_name(ins, "default", (uint32_t)strlen("default"));
if (ts_node_is_null(nm)) {
return;
}
const char *name = dk_text(d, nm);
if (!name) {
return;
}
if (d->stage < 0) {
dk_set(d, &d->global, name, ts_node_is_null(df) ? NULL : dk_value(d, df));
return;
}
const char *value = NULL;
if (!ts_node_is_null(df)) {
const char *raw = dk_value(d, df);
value = raw ? dk_expand(d, d->local, raw) : NULL;
} else {
const dk_var_t *g = dk_find(d->global, name, strlen(name));
value = g ? g->value : NULL;
}
dk_set(d, &d->local, name, value);
}
static void dk_env(dk_t *d, TSNode ins) {
if (d->stage < 0) {
return;
}
for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) {
TSNode pair = ts_node_named_child(ins, i);
if (!dk_type(pair, "env_pair")) {
continue;
}
TSNode nm = ts_node_child_by_field_name(pair, "name", (uint32_t)strlen("name"));
TSNode vl = ts_node_child_by_field_name(pair, "value", (uint32_t)strlen("value"));
if (ts_node_is_null(nm)) {
continue;
}
const char *raw = ts_node_is_null(vl) ? NULL : dk_value(d, vl);
dk_set(d, &d->local, dk_text(d, nm), raw ? dk_expand(d, d->local, raw) : NULL);
}
}
static void dk_from(dk_t *d, TSNode ins) {
TSNode spec = {0};
TSNode alias = ts_node_child_by_field_name(ins, "as", (uint32_t)strlen("as"));
for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) {
if (dk_type(ts_node_named_child(ins, i), "image_spec")) {
spec = ts_node_named_child(ins, i);
}
}
const char *ref = ts_node_is_null(spec) ? NULL : dk_expand(d, d->global, dk_text(d, spec));
d->stage++;
d->local = NULL;
if (ref && !dk_is_stage(d, ref)) {
dk_link(d, ref, "from", dk_line(ins));
}
if (d->stage < DK_MAX_STAGES) {
d->stages[d->stage] = ts_node_is_null(alias) ? NULL : dk_text(d, alias);
}
}
static void dk_copy(dk_t *d, TSNode ins) {
static const char FROM[] = "--from=";
for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) {
TSNode p = ts_node_named_child(ins, i);
if (!dk_type(p, "param")) {
continue;
}
const char *t = dk_text(d, p);
if (!t || strncmp(t, FROM, sizeof(FROM) - SKIP_ONE) != 0) {
continue;
}
const char *ref = dk_expand(d, d->local, t + sizeof(FROM) - SKIP_ONE);
if (ref && ref[0] && !dk_is_stage(d, ref)) {
dk_link(d, ref, "copy-from", dk_line(ins));
}
}
}
static void dk_file(dk_t *d, TSNode root) {
for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) {
TSNode ins = ts_node_named_child(root, i);
if (dk_type(ins, "arg_instruction")) {
dk_arg(d, ins);
} else if (dk_type(ins, "env_instruction")) {
dk_env(d, ins);
} else if (dk_type(ins, "from_instruction")) {
dk_from(d, ins);
} else if (dk_type(ins, "copy_instruction") && d->stage >= 0) {
dk_copy(d, ins);
}
}
}
static char *dk_read(const char *path, size_t *len) {
FILE *f = cbm_fopen(path, "rb");
if (!f) {
return NULL;
}
(void)fseek(f, 0, SEEK_END);
long size = ftell(f);
(void)fseek(f, 0, SEEK_SET);
if (size <= 0 || size > cbm_max_file_bytes()) {
(void)fclose(f);
return NULL;
}
enum { DK_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */
char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + DK_TS_PAD);
if (!buf) {
(void)fclose(f);
return NULL;
}
size_t got = fread(buf, SKIP_ONE, (size_t)size, f);
(void)fclose(f);
memset(buf + got, 0, DK_TS_PAD);
*len = got;
return buf;
}
void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
int file_count) {
TSParser *parser = NULL;
CBMArena arena;
cbm_arena_init_lazy(&arena, CBM_SZ_4K);
int edges = 0;
for (int i = 0; i < file_count; i++) {
if (files[i].language != CBM_LANG_DOCKERFILE) {
continue;
}
const char *mqn = cbm_fqn_module(&arena, ctx->project_name, files[i].rel_path);
const cbm_gbuf_node_t *mod = mqn ? cbm_gbuf_find_by_qn(ctx->gbuf, mqn) : NULL;
if (!mod) {
continue;
}
size_t len = 0;
char *src = dk_read(files[i].path, &len);
if (!src) {
continue;
}
if (!parser) {
parser = ts_parser_new();
}
TSTree *tree = NULL;
if (parser && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_DOCKERFILE))) {
tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len);
}
if (tree) {
dk_t d = {.ctx = ctx,
.a = &arena,
.src = src,
.file = files[i].rel_path,
.module = mod->id,
.stage = -1};
dk_file(&d, ts_tree_root_node(tree));
edges += d.edges;
ts_tree_delete(tree);
}
cbm_free(CBM_MEM_CLASS_OTHER, src);
}
if (parser) {
ts_parser_delete(parser);
}
cbm_arena_destroy(&arena);
if (edges > 0) {
char buf[CBM_SZ_32];
snprintf(buf, sizeof(buf), "%d", edges);
cbm_log_info("pass.docker.images", "edges", buf);
}
}
+1
View File
@@ -3471,6 +3471,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
/* Shell and Make deploy commands, against the nodes linked above. */
cbm_pipeline_pass_shell(ctx, files, file_count);
cbm_pipeline_pass_terraform(ctx, files, file_count);
cbm_pipeline_pass_docker(ctx, files, file_count);
cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count));
(void)helm_count;
+22 -8
View File
@@ -248,22 +248,36 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *
}
int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label,
const char *edge_type) {
const char *const *edge_types) {
sqlite3 *db = cbm_store_get_db(store);
if (!db) {
if (!db || !edge_types) {
return CBM_NOT_FOUND;
}
/* One statement: a pair of NOT EXISTS per listed type (target, source),
* each served by an (id, type) index. */
char sql[CBM_SZ_2K];
int len = snprintf(sql, sizeof(sql), "DELETE FROM nodes WHERE project = ?1 AND label = ?2");
int t = 0;
for (; edge_types[t] && len > 0 && (size_t)len < sizeof(sql); t++) {
len += snprintf(sql + len, sizeof(sql) - (size_t)len,
" AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.project = ?1"
" AND e.target_id = nodes.id AND e.type = ?%d)"
" AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.project = ?1"
" AND e.source_id = nodes.id AND e.type = ?%d)",
t + 3, t + 3);
}
if (t == 0 || len <= 0 || (size_t)len >= sizeof(sql)) {
return CBM_NOT_FOUND;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db,
"DELETE FROM nodes WHERE project = ?1 AND label = ?2 AND NOT EXISTS"
" (SELECT 1 FROM edges e WHERE e.project = ?1"
" AND e.target_id = nodes.id AND e.type = ?3)",
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
return CBM_NOT_FOUND;
}
sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 3, edge_type, CBM_NOT_FOUND, SQLITE_TRANSIENT);
for (int i = 0; i < t; i++) {
sqlite3_bind_text(stmt, i + 3, edge_types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT);
}
int rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND;
+9 -6
View File
@@ -3087,8 +3087,9 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
/* The proxy buffer holds no edges, so only the patched staging graph can
* tell a Process node no SPAWNS reaches any more (its last spawner was
* purged). */
if (cbm_delta_drop_unreferenced(staging, project, "Process", "SPAWNS") < 0) {
* purged), or an Image nothing builds on, builds or runs. */
if (cbm_delta_drop_unreferenced(staging, project, "Process", SH_PROCESS_EDGES) < 0 ||
cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0) {
goto out;
}
@@ -3697,10 +3698,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
}
if (phase_rc == 0) {
/* A Process node exists only as a SPAWNS target (pass_spawns.c,
* pass_shell.c) and carries no file: when the last file starting the
* program is gone, nothing else removes it -- a full index never
* makes it. This buffer holds the whole graph. */
(void)cbm_gbuf_delete_unreferenced(existing, "Process", "SPAWNS");
* pass_shell.c), an Image only through its DEPENDS_ON / USAGE edges
* (pass_docker.c); neither carries a file: when the last file naming
* one is gone, nothing else removes it -- a full index never makes
* it. This buffer holds the whole graph. */
(void)cbm_gbuf_delete_unreferenced(existing, "Process", SH_PROCESS_EDGES);
(void)cbm_gbuf_delete_unreferenced(existing, "Image", DK_IMAGE_EDGES);
phase_rc = run_postpasses(&ctx, changed_files, ci, project, true);
}
+13 -3
View File
@@ -1079,6 +1079,11 @@ void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len,
void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
int file_count);
/* Container builds (pass_docker.c): a Dockerfile DEPENDS_ON the Images it
* builds on (FROM, COPY --from). Run at the end of the K8s pass. */
void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
int file_count);
/* Pre-dump pass: markdown → file REFERENCES_FILE linking. */
int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx);
@@ -1290,10 +1295,15 @@ int cbm_delta_snapshot_inbound(cbm_store_t *store, const char *project, const ch
void cbm_delta_free_snapshot(cbm_delta_saved_edge_t *items, int count);
int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *paths,
int path_count);
/* Delete the project's `label` nodes no `edge_type` edge targets (a Process
* whose last spawner was purged); returns how many, or -1 on failure. */
/* Delete the project's `label` nodes no edge of `edge_types` (NULL-terminated)
* touches as source or target (a Process whose last spawner was purged, an
* Image no Dockerfile or workload names any more); returns how many, or -1
* on failure. */
int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label,
const char *edge_type);
const char *const *edge_types);
/* The edges that keep a file-less shared node alive. */
#define SH_PROCESS_EDGES ((const char *const[]){"SPAWNS", NULL})
#define DK_IMAGE_EDGES ((const char *const[]){"DEPENDS_ON", "USAGE", NULL})
/* Pre-seed proxy nodes with their REAL database ids and move the gbuf id
* watermark above MAX(id); returns that max id, or -1 on failure. */
int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf);
+2 -1
View File
@@ -28,7 +28,8 @@ typedef struct cbm_store_read_scope cbm_store_read_scope_t;
* rebuild (pipeline.c, format_change_reindex) that keeps ADR/project metadata.
* 1 #1108: File-node QNs keep the file extension.
* 2 Kubernetes objects: one Resource per YAML document, QN
* <file>::<group>/<Kind>/<declared namespace or ->/<name>. */
* <file>::<group>/<Kind>/<declared namespace or ->/<name>; a
* Dockerfile.<variant> keeps its suffix in its QNs. */
#define CBM_INDEX_FORMAT_VERSION 2
#define CBM_STORE_CANCELLED (-3)
#define CBM_STORE_SCAN_LIMIT (-4)
+22
View File
@@ -7,6 +7,8 @@
#include "test_framework.h"
#include "../src/pipeline/pipeline.h"
#include "../src/foundation/str_util.h"
#include "../src/foundation/arena.h"
#include "helpers.h"
#include <stdlib.h>
#include <string.h>
@@ -27,6 +29,25 @@
/* ── Basic: project + path + name ─────────────────────────────── */
/* A Dockerfile variant keeps its suffix in both QN implementations (the
* pipeline's fqn.c, the extractor's helpers.c) -- else it gets the QN of the
* plain Dockerfile beside it. */
TEST(fqn_dockerfile_variant_keeps_suffix) {
ASSERT_FQN(cbm_pipeline_fqn_compute("p", "svc/Dockerfile.debug", "GOOS"),
"p.svc.Dockerfile.debug.GOOS");
ASSERT_FQN(cbm_pipeline_fqn_module("p", "svc/Containerfile.arm64"), "p.svc.Containerfile.arm64");
ASSERT_FQN(cbm_pipeline_fqn_module("p", "svc/Dockerfile"), "p.svc.Dockerfile");
ASSERT_FQN(cbm_pipeline_fqn_module("p", "svc/app.py"), "p.svc.app");
CBMArena a;
cbm_arena_init(&a);
ASSERT_STR_EQ(cbm_fqn_compute(&a, "p", "svc/Dockerfile.debug", "GOOS"),
"p.svc.Dockerfile.debug.GOOS");
ASSERT_STR_EQ(cbm_fqn_module(&a, "p", "svc/Containerfile.arm64"), "p.svc.Containerfile.arm64");
ASSERT_STR_EQ(cbm_fqn_module(&a, "p", "svc/app.py"), "p.svc.app");
cbm_arena_destroy(&a);
PASS();
}
TEST(fqn_compute_basic_go) {
ASSERT_FQN(cbm_pipeline_fqn_compute("myproj", "main.go", "main"), "myproj.main.main");
PASS();
@@ -673,6 +694,7 @@ TEST(project_name_length_capped_issue624) {
SUITE(fqn) {
/* fqn_compute: basic extensions */
RUN_TEST(fqn_compute_basic_go);
RUN_TEST(fqn_dockerfile_variant_keeps_suffix);
RUN_TEST(fqn_compute_basic_py);
RUN_TEST(fqn_compute_basic_ts);
RUN_TEST(fqn_compute_basic_js);
+85
View File
@@ -1880,6 +1880,89 @@ static char *gpf_runs(const GpfFile *files, int nfiles) {
free(got_); \
} while (0)
/* Every DEPENDS_ON edge into an Image as "@<file> -> <repository> (via)",
* sorted, one per line. */
static char *gpf_images(const GpfFile *files, int nfiles) {
GpfProj lp;
cbm_store_t *store = gpf_index_files(&lp, files, nfiles);
char *out = NULL;
if (store) {
sqlite3_stmt *st = NULL;
const char *sql =
"SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT '@' || s.file_path "
"|| ' -> ' || t.name || ' (' || json_extract(e.properties,'$.via') || ')' AS x "
"FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id "
"WHERE e.project=?1 AND e.type='DEPENDS_ON' AND t.label='Image' ORDER BY x)";
if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC);
if (sqlite3_step(st) == SQLITE_ROW) {
out = strdup((const char *)sqlite3_column_text(st, 0));
}
}
sqlite3_finalize(st);
}
gpf_cleanup(&lp, store);
return out;
}
#define GPF_ASSERT_IMAGES(files, n, want) \
do { \
char *got_ = gpf_images(files, n); \
if (!got_ || strcmp(got_, want) != 0) { \
fprintf(stderr, "images got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \
free(got_); \
FAIL("Image DEPENDS_ON edge set"); \
} \
free(got_); \
} while (0)
/* A Dockerfile DEPENDS_ON each image it builds on: FROM and COPY --from,
* with ARG defaults expanded, under the image's repository (docker.io and
* library/ filled in, a registry host kept, tag and digest dropped). A stage
* (by name or index), scratch and an ARG without a value are no image. */
TEST(probe_dockerfile_bases_are_images) {
static const GpfFile files[] = {
{"svc/Dockerfile", "ARG GO=1.22\nARG BASE\n"
"FROM golang:${GO} AS build\n"
"RUN go build -o /out/app ./cmd/app\n"
"FROM ${BASE} AS unknown\n"
"FROM build AS test\n"
"FROM gcr.io/distroless/static@sha256:0123 AS run\n"
"COPY --from=build /out/app /app\n"
"COPY --from=0 /out/app /app2\n"
"COPY --from=docker.io/bitnami/kubectl:1.30 /opt/kubectl /kubectl\n"
"FROM scratch\n"
"COPY --from=registry.example.com:5000/team/tools:2 /bin/x /x\n"}};
GPF_ASSERT_IMAGES(files, 1,
"@svc/Dockerfile -> docker.io/bitnami/kubectl (copy-from)\n"
"@svc/Dockerfile -> docker.io/library/golang (from)\n"
"@svc/Dockerfile -> gcr.io/distroless/static (from)\n"
"@svc/Dockerfile -> registry.example.com:5000/team/tools (copy-from)");
PASS();
}
/* A build file named for its variant is a Dockerfile of its own (its own
* Module, beside the plain Dockerfile): Dockerfile.debug, Dockerfile-test,
* Containerfile, pipeline.Dockerfile. A template of one (Dockerfile.j2) and a
* BuildKit ignore file (Dockerfile.dockerignore) are not. */
TEST(probe_dockerfile_variants_are_dockerfiles) {
static const GpfFile files[] = {
{"svc/Dockerfile", "FROM alpine:3.20\n"},
{"svc/Dockerfile.debug", "FROM debian:12\n"},
{"svc/Dockerfile-test", "FROM ubuntu:24.04\n"},
{"svc/Dockerfile.dockerignore", "FROM busybox\n"},
{"Containerfile", "FROM quay.io/podman/stable\n"},
{"tools/pipeline.Dockerfile", "FROM node:20\n"},
{"tools/Dockerfile.j2", "FROM {{ base_image }}\n"}};
GPF_ASSERT_IMAGES(files, 7,
"@Containerfile -> quay.io/podman/stable (from)\n"
"@svc/Dockerfile -> docker.io/library/alpine (from)\n"
"@svc/Dockerfile-test -> docker.io/library/ubuntu (from)\n"
"@svc/Dockerfile.debug -> docker.io/library/debian (from)\n"
"@tools/pipeline.Dockerfile -> docker.io/library/node (from)");
PASS();
}
#define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n"
#define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"
@@ -4143,6 +4226,8 @@ SUITE(grammar_probe_f) {
RUN_TEST(probe_mcp_trace_topology);
RUN_TEST(probe_mcp_trace_topology_follows_deploys);
RUN_TEST(probe_mcp_trace_deploy);
RUN_TEST(probe_dockerfile_bases_are_images);
RUN_TEST(probe_dockerfile_variants_are_dockerfiles);
RUN_TEST(probe_gitops_objects_deploy_their_units);
RUN_TEST(probe_shell_kubectl_deploys_what_it_applies);
RUN_TEST(probe_shell_helm_deploys_its_chart);
+16
View File
@@ -549,6 +549,21 @@ TEST(lang_fn_dockerfile) {
ASSERT_EQ(cbm_language_for_filename("Dockerfile"), CBM_LANG_DOCKERFILE);
PASS();
}
/* Build files named for a variant; a template or ignore file of one is not. */
TEST(lang_fn_dockerfile_variants) {
ASSERT_EQ(cbm_language_for_filename("Dockerfile.debug"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("Dockerfile.multiarch"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("Dockerfile-test"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("Dockerfile_base"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("Containerfile"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("Containerfile.arm64"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("pipeline.Dockerfile"), CBM_LANG_DOCKERFILE);
ASSERT_EQ(cbm_language_for_filename("Dockerfile.j2"), CBM_LANG_JINJA2);
ASSERT_NEQ(cbm_language_for_filename("Dockerfile.dockerignore"), CBM_LANG_DOCKERFILE);
ASSERT_NEQ(cbm_language_for_filename("Dockerfile."), CBM_LANG_DOCKERFILE);
ASSERT_NEQ(cbm_language_for_filename("Dockerfiles"), CBM_LANG_DOCKERFILE);
PASS();
}
TEST(lang_fn_meson_build) {
ASSERT_EQ(cbm_language_for_filename("meson.build"), CBM_LANG_MESON);
PASS();
@@ -1635,6 +1650,7 @@ SUITE(language) {
RUN_TEST(lang_fn_makefile_lower);
RUN_TEST(lang_fn_cmake);
RUN_TEST(lang_fn_dockerfile);
RUN_TEST(lang_fn_dockerfile_variants);
RUN_TEST(lang_fn_meson_build);
RUN_TEST(lang_fn_meson_opts);
RUN_TEST(lang_fn_meson_opts_txt);
+47
View File
@@ -3508,6 +3508,52 @@ TEST(incremental_deleted_spawner_drops_its_process) {
PASS();
}
/* Deleting the only Dockerfile naming an image leaves no Image node behind;
* the image another Dockerfile still names stays. `legacy`: through the
* full-load route, else the closure-repair route. Returns the Image count
* after the deletion, or -1; *route: the route taken. */
static int incremental_dockerfile_delete(bool legacy, cbm_incremental_route_t *route) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_imagegc_XXXXXX");
if (!cbm_mkdtemp(tmp)) {
return -1;
}
write_temp_file(tmp, "a/Dockerfile",
"FROM alpine:3.20\nCOPY --from=docker.io/bitnami/kubectl:1.30 /k /k\n");
write_temp_file(tmp, "b/Dockerfile", "FROM alpine:3.19\n");
char db_path[512];
snprintf(db_path, sizeof(db_path), "%s/imagegc.db", tmp);
static const char images[] = "SELECT COUNT(*) FROM nodes WHERE label='Image'";
int full = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, images) : -1;
char path[512];
snprintf(path, sizeof(path), "%s/a/Dockerfile", tmp);
(void)remove(path);
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
if (legacy) {
cbm_pipeline_incremental_test_force_legacy_partial_once();
}
#else
(void)legacy;
#endif
int after = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, images) : -1;
*route = cbm_pipeline_incremental_test_last_route();
th_rmtree(tmp);
return full == 2 ? after : -1;
}
TEST(incremental_deleted_dockerfile_drops_its_image) {
cbm_incremental_route_t route = CBM_INCREMENTAL_ROUTE_NONE;
int left = incremental_dockerfile_delete(false, &route);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
ASSERT_EQ(left, 1);
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
left = incremental_dockerfile_delete(true, &route);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL);
ASSERT_EQ(left, 1);
#endif
PASS();
}
/* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage,
* while the shared name registry must NOT leak those relations into other
* languages' textual resolution: a Python call or identifier sharing the
@@ -19134,6 +19180,7 @@ SUITE(pipeline_semantic_manifest_repro) {
RUN_TEST(incremental_k8s_edits_reach_other_files);
RUN_TEST(incremental_shell_deploy_edit_keeps_its_edges);
RUN_TEST(incremental_deleted_spawner_drops_its_process);
RUN_TEST(incremental_deleted_dockerfile_drops_its_image);
RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
RUN_TEST(pipeline_dbt_jinja_lineage);