feat(docker): a repository built per component splits into component Images

An image is its repository, tag and digest on the edges -- until one
repository is built from several Dockerfiles, one per component tag:
opentelemetry-demo builds every service as
ghcr.io/open-telemetry/demo:<version>-<service>, so its one Image was
built from 25 Dockerfiles and a walk from any service through the image
reached all of them.

Such a repository is now split by component: its build tags without
their common prefix / suffix (cut at separators -- latest-frontend,
latest-cart -> frontend, cart). A component built from exactly one
Dockerfile, with no unknown part, becomes an Image of its own,
`<repo>:*-<component>` {repository, component, tag_pattern}; its build
edges leave from it, and every reference whose tag names a component (a
K8s container, a compose service, a FROM) uses it -- whatever its
version, the longest component a tag holds as whole parts, none on a
tie. Everything else stays on the repository Image, which goes when no
edge is left (now on the full route too). Recomputed whole each run.

A closure repair cannot re-point the references of files it does not
repair, so when the run's component Images differ from the previous
ones it hands over to a full rebuild.

Edges are deleted by pointer (cbm_gbuf_delete_edges): after the
parallel resolve merge, cbm_gbuf_set_next_id can hand an edge id out
twice, and a delete by id took unrelated edges with it.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-10-11 18:49:18 +02:00
1 parent 4ed6064c31
commit 231bcbf6f5
9 files changed
+734 -6

No files matched your search

+28
View File
@@ -1919,6 +1919,34 @@ int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const cha
return gbuf_delete_edges_if(gb, gbuf_edge_from_label, &tl);
}
static int gbuf_ptr_cmp(const void *k, const void *v) {
uintptr_t a = (uintptr_t)*(const cbm_gbuf_edge_t *const *)k;
uintptr_t b = (uintptr_t)*(const cbm_gbuf_edge_t *const *)v;
return (a > b) - (a < b);
}
typedef struct {
const cbm_gbuf_edge_t **edges;
int n;
} gbuf_edge_set_t;
static bool gbuf_edge_in_set(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) {
(void)gb;
const gbuf_edge_set_t *s = ud;
return bsearch(&e, s->edges, (size_t)s->n, sizeof(*s->edges), gbuf_ptr_cmp) != NULL;
}
/* By the edges themselves, not their ids: after the parallel resolve merge,
* cbm_gbuf_set_next_id can hand an id out twice. */
int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n) {
if (!gb || !edges || n <= 0) {
return 0;
}
qsort((void *)edges, (size_t)n, sizeof(*edges), gbuf_ptr_cmp);
gbuf_edge_set_t s = {edges, n};
return gbuf_delete_edges_if(gb, gbuf_edge_in_set, &s);
}
/* ── Merge ───────────────────────────────────────────────────────── */
/* Free remap hash table entries (key = heap string, value = heap int64_t*) */
+4
View File
@@ -162,6 +162,10 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type);
/* Delete the edges of `type` whose source node has label `label`. */
int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const char *label);
/* Delete edges[0..n) (pointers found in this buffer; the array is sorted in
* place). */
int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n);
/* ── Vector storage (for semantic embeddings) ───────────────────── */
/* Store an int8-quantized vector for a node. The vector data is copied.
+434
View File
@@ -54,6 +54,7 @@
#include "lang_specs.h"
#include <ctype.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
@@ -1465,6 +1466,438 @@ static void dk_index_build(cbm_pipeline_ctx_t *ctx, CBMArena *a, dk_index_t *ix)
}
}
/* ── Image tag sub-identity (P4.2b) ─────────────────────────────── */
_Atomic uint64_t g_dk_hub_edges = 0; /* complexity guard: Image edges a run examines */
/* A plain JSON string property (an image reference holds no escapes). */
static bool dk_prop(const char *props, const char *key, char *out, size_t out_sz) {
char pat[CBM_SZ_64];
snprintf(pat, sizeof(pat), "\"%s\":\"", key);
const char *p = props ? strstr(props, pat) : NULL;
if (!p) {
return false;
}
p += strlen(pat);
const char *e = strchr(p, '"');
if (!e || (size_t)(e - p) >= out_sz || memchr(p, '\\', (size_t)(e - p))) {
return false;
}
memcpy(out, p, (size_t)(e - p));
out[e - p] = '\0';
return true;
}
/* The tag of image reference `ref`: after the ':' that follows its last
* '/' (a registry port is no tag), up to a '@' digest; "" for none. */
static void dk_tag(const char *ref, char *out, size_t sz) {
out[0] = '\0';
const char *slash = strrchr(ref, '/');
const char *colon = strchr(slash ? slash : ref, ':');
if (colon) {
snprintf(out, sz, "%.*s", (int)strcspn(colon + SKIP_ONE, "@"), colon + SKIP_ONE);
}
}
/* The repository of an Image node's name ("<repo>" or "<repo>:<pattern>"). */
static void dk_repo_of(const char *name, char *out, size_t sz) {
const char *slash = strrchr(name, '/');
const char *colon = strchr(slash ? slash : name, ':');
snprintf(out, sz, "%.*s", (int)(colon ? (size_t)(colon - name) : strlen(name)), name);
}
static bool dk_sep(char c) {
return c == '-' || c == '_' || c == '.';
}
/* Does `comp` occur in `tag` as whole separator-delimited parts? */
static bool dk_tag_has(const char *tag, const char *comp) {
size_t cl = strlen(comp);
for (const char *p = strstr(tag, comp); p; p = strstr(p + SKIP_ONE, comp)) {
if ((p == tag || dk_sep(p[-SKIP_ONE])) && (p[cl] == '\0' || dk_sep(p[cl]))) {
return true;
}
}
return false;
}
/* An edge of a repository's Images, copied before any edge moves. */
typedef struct {
const cbm_gbuf_edge_t *edge; /* stable until deleted */
int64_t src;
int64_t tgt;
const char *type;
const char *props;
const char *tag;
int comp; /* its component; -1: none */
} dk_hub_edge_t;
typedef struct {
dk_hub_edge_t *v;
int n;
int cap;
} dk_hub_edges_t;
typedef struct {
const cbm_gbuf_edge_t **v;
int n;
int cap;
} dk_drop_t;
typedef struct {
int64_t id;
const char *name;
const char *repo;
} dk_img_t;
static int dk_img_cmp(const void *x, const void *y) {
const dk_img_t *a = x;
const dk_img_t *b = y;
int c = strcmp(a->repo, b->repo);
return c ? c : strcmp(a->name, b->name);
}
/* Grow an arena array of `n` items of `size` to hold one more. */
static void *dk_grow(CBMArena *a, void *v, int n, int *cap, size_t size) {
if (n < *cap) {
return v;
}
int nc = *cap ? *cap * PAIR_LEN : CBM_SZ_16;
void *g = cbm_arena_alloc(a, (size_t)nc * size);
if (g && n) {
memcpy(g, v, (size_t)n * size);
}
if (g) {
*cap = nc;
}
return g;
}
static void dk_hub_push(CBMArena *a, dk_hub_edges_t *l, const cbm_gbuf_edge_t *e, const char *tag) {
dk_hub_edge_t *v = dk_grow(a, l->v, l->n, &l->cap, sizeof(*v));
if (!v) {
return;
}
l->v = v;
l->v[l->n++] = (dk_hub_edge_t){e,
e->source_id,
e->target_id,
cbm_arena_strdup(a, e->type),
cbm_arena_strdup(a, e->properties_json),
cbm_arena_strdup(a, tag),
-1};
}
/* Is an edge of `type` with properties `p` a reference to an image --
* USAGE {ref:image}, or DEPENDS_ON {via:from|copy-from}? */
static bool dk_hub_ref(const char *type, const char *p) {
char v[CBM_SZ_32];
if (strcmp(type, "USAGE") == 0) {
return dk_prop(p, "ref", v, sizeof(v)) && strcmp(v, "image") == 0;
}
return dk_prop(p, "via", v, sizeof(v)) &&
(strcmp(v, "from") == 0 || strcmp(v, "copy-from") == 0);
}
/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}) and
* references in, each with the tag of its `image` reference. */
static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img,
dk_hub_edges_t *builds, dk_hub_edges_t *refs) {
const cbm_gbuf_edge_t **es = NULL;
int n = 0;
char ref[CBM_SZ_1K];
char tag[CBM_SZ_256];
if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, img, "DEPENDS_ON", &es, &n) == 0) {
for (int i = 0; i < n; i++) {
char via[CBM_SZ_32];
if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via)) ||
strcmp(via, "build") != 0) {
continue;
}
tag[0] = '\0';
if (dk_prop(es[i]->properties_json, "image", ref, sizeof(ref))) {
dk_tag(ref, tag, sizeof(tag));
}
dk_hub_push(a, builds, es[i], tag);
}
atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed);
}
static const char *const IN[] = {"USAGE", "DEPENDS_ON"};
for (size_t t = 0; t < sizeof(IN) / sizeof(IN[0]); t++) {
if (cbm_gbuf_find_edges_by_target_type(ctx->gbuf, img, IN[t], &es, &n) != 0) {
continue;
}
atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed);
for (int i = 0; i < n; i++) {
if (dk_hub_ref(IN[t], es[i]->properties_json) &&
dk_prop(es[i]->properties_json, "image", ref, sizeof(ref))) {
dk_tag(ref, tag, sizeof(tag));
dk_hub_push(a, refs, es[i], tag);
}
}
}
}
/* The number of distinct Dockerfiles the build edges target. */
static int dk_hub_dockerfiles(CBMArena *a, const dk_hub_edges_t *b) {
CBMHashTable *seen = cbm_ht_create(0);
int n = 0;
for (int i = 0; seen && i < b->n; i++) {
const char *k = cbm_arena_sprintf(a, "%lld", (long long)b->v[i].tgt);
if (k && !cbm_ht_has(seen, k)) {
cbm_ht_set(seen, k, (void *)SKIP_ONE);
n++;
}
}
if (seen) {
cbm_ht_free(seen);
}
return n;
}
/* The common prefix / suffix of the build tags, cut back to a separator and
* leaving every tag a non-empty middle where it can: *pre, *suf. */
static void dk_hub_affixes(const dk_hub_edges_t *b, size_t *pre, size_t *suf) {
const char *t0 = b->v[0].tag;
size_t l0 = strlen(t0);
size_t p = l0;
size_t s = l0;
for (int i = SKIP_ONE; i < b->n; i++) {
const char *t = b->v[i].tag;
size_t tl = strlen(t);
size_t k = 0;
while (k < p && k < tl && t[k] == t0[k]) {
k++;
}
p = k;
k = 0;
while (k < s && k < tl && t[tl - SKIP_ONE - k] == t0[l0 - SKIP_ONE - k]) {
k++;
}
s = k;
}
while (p > 0 && !dk_sep(t0[p - SKIP_ONE])) {
p--;
}
while (s > 0 && !dk_sep(t0[l0 - s])) {
s--;
}
for (int i = 0; i < b->n; i++) {
size_t tl = strlen(b->v[i].tag);
while (s > 0 && p + s >= tl) {
for (s--; s > 0 && !dk_sep(t0[l0 - s]); s--) {}
}
while (p > 0 && p + s >= tl) {
for (p--; p > 0 && !dk_sep(t0[p - SKIP_ONE]); p--) {}
}
}
*pre = p;
*suf = s;
}
/* Components of a repository built from several Dockerfiles: each build
* tag without the tags' common prefix / suffix (otel's latest-frontend,
* latest-cart -> frontend, cart). A component built from exactly one
* Dockerfile, holding no unknown part, is kept (comps[k] non-NULL); each
* build edge's comp is set. Returns the number of component slots (0:
* none kept). */
static int dk_hub_components(CBMArena *a, dk_hub_edges_t *b, size_t pre, size_t suf,
const char ***comps) {
const char **names = cbm_arena_alloc(a, (size_t)b->n * sizeof(*names));
int64_t *dfs = cbm_arena_alloc(a, (size_t)b->n * sizeof(*dfs));
CBMHashTable *index = cbm_ht_create(0); /* component -> slot + 1 */
int nc = 0;
for (int i = 0; names && dfs && index && i < b->n; i++) {
const char *t = b->v[i].tag;
size_t tl = strlen(t);
const char *c = tl > pre + suf ? cbm_arena_strndup(a, t + pre, tl - pre - suf) : NULL;
if (!c || strpbrk(c, "$*{}()`\"\\")) {
continue;
}
int k = (int)(intptr_t)cbm_ht_get(index, c) - SKIP_ONE;
if (k < 0) {
k = nc++;
names[k] = c;
dfs[k] = b->v[i].tgt;
cbm_ht_set(index, c, (void *)(intptr_t)(k + SKIP_ONE));
} else if (names[k] && dfs[k] != b->v[i].tgt) {
names[k] = NULL; /* one name, two Dockerfiles: no identity */
}
b->v[i].comp = k;
}
if (index) {
cbm_ht_free(index);
}
if (!names || !dfs) {
return 0;
}
int kept = 0;
for (int k = 0; k < nc; k++) {
kept += names[k] ? SKIP_ONE : 0;
}
for (int i = 0; i < b->n; i++) {
if (b->v[i].comp >= 0 && !names[b->v[i].comp]) {
b->v[i].comp = -1;
}
}
*comps = names;
return kept ? nc : 0;
}
/* The component a reference's tag names: the longest kept one occurring in
* it as whole parts; -1 for none, or for two of that length. */
static int dk_hub_match(const char *tag, const char *const *comps, int nc) {
int best = -1;
size_t bl = 0;
bool tie = false;
for (int k = 0; k < nc; k++) {
if (!comps[k] || !dk_tag_has(tag, comps[k])) {
continue;
}
size_t l = strlen(comps[k]);
if (l > bl) {
best = k;
bl = l;
tie = false;
} else if (l == bl) {
tie = true;
}
}
return tie ? -1 : best;
}
/* The Image of a component: `<repo>:<pattern>` (`*-frontend`: the
* prefix's version masked) {repository, component, tag_pattern}. */
static int64_t dk_hub_image(cbm_pipeline_ctx_t *ctx, const char *repo, const char *comp,
char pre_sep, char suf_sep) {
char head[PAIR_LEN + SKIP_ONE] = {0};
char tail[PAIR_LEN + SKIP_ONE] = {0};
if (pre_sep) {
head[0] = '*';
head[SKIP_ONE] = pre_sep;
}
if (suf_sep) {
tail[0] = suf_sep;
tail[SKIP_ONE] = '*';
}
char pattern[CBM_SZ_512];
char name[CBM_SZ_1K];
char qn[CBM_SZ_2K];
snprintf(pattern, sizeof(pattern), "%s%s%s", head, comp, tail);
snprintf(name, sizeof(name), "%s:%s", repo, pattern);
snprintf(qn, sizeof(qn), "__image__%s", name);
char esc_r[CBM_SZ_1K];
char esc_c[CBM_SZ_1K];
char esc_p[CBM_SZ_1K];
cbm_json_escape(esc_r, (int)sizeof(esc_r), repo);
cbm_json_escape(esc_c, (int)sizeof(esc_c), comp);
cbm_json_escape(esc_p, (int)sizeof(esc_p), pattern);
char props[CBM_SZ_4K];
snprintf(props, sizeof(props),
"{\"repository\":\"%s\",\"component\":\"%s\",\"tag_pattern\":\"%s\"}", esc_r, esc_c,
esc_p);
return cbm_gbuf_upsert_node(ctx->gbuf, "Image", name, qn, "", 0, 0, props);
}
/* Move edge `e` to (src, tgt): insert the copy, queue `e` for deletion. */
static int dk_hub_move(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_hub_edge_t *e, int64_t src,
int64_t tgt, dk_drop_t *drop) {
if ((src == e->src && tgt == e->tgt) || src <= 0 || tgt <= 0) {
return 0;
}
const cbm_gbuf_edge_t **v = dk_grow(a, (void *)drop->v, drop->n, &drop->cap, sizeof(*v));
if (!v) {
return 0;
}
drop->v = v;
drop->v[drop->n++] = e->edge;
cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, e->type, e->props);
return SKIP_ONE;
}
/* One repository's Images (imgs[0..n)): split it by component when it is
* built from several Dockerfiles, else keep (or bring back) everything on
* the repository Image. Returns the edges moved. */
static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *imgs, int n,
dk_drop_t *drop) {
dk_hub_edges_t builds = {0};
dk_hub_edges_t refs = {0};
bool has_sub = false;
for (int i = 0; i < n; i++) {
has_sub = has_sub || strcmp(imgs[i].name, imgs[i].repo) != 0;
dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs);
}
const char **comps = NULL;
size_t pre = 0;
size_t suf = 0;
int nc = 0;
if (builds.n > SKIP_ONE && dk_hub_dockerfiles(a, &builds) > SKIP_ONE) {
dk_hub_affixes(&builds, &pre, &suf);
nc = dk_hub_components(a, &builds, pre, suf, &comps);
}
if (nc == 0 && !has_sub) {
return 0;
}
int64_t repo_img = cbm_pipeline_image_node(ctx, imgs[0].repo);
int64_t *sub = nc ? cbm_arena_alloc(a, (size_t)nc * sizeof(*sub)) : NULL;
const char *t0 = builds.n ? builds.v[0].tag : "";
char pre_sep = pre ? t0[pre - SKIP_ONE] : '\0';
char suf_sep = suf ? t0[strlen(t0) - suf] : '\0';
for (int k = 0; k < nc && sub; k++) {
sub[k] = comps[k] ? dk_hub_image(ctx, imgs[0].repo, comps[k], pre_sep, suf_sep) : 0;
}
int moved = 0;
for (int i = 0; i < builds.n; i++) {
int k = builds.v[i].comp;
int64_t src = sub && k >= 0 && sub[k] > 0 ? sub[k] : repo_img;
moved += dk_hub_move(ctx, a, &builds.v[i], src, builds.v[i].tgt, drop);
}
for (int i = 0; i < refs.n; i++) {
int k = sub ? dk_hub_match(refs.v[i].tag, comps, nc) : -1;
int64_t tgt = k >= 0 && sub[k] > 0 ? sub[k] : repo_img;
moved += dk_hub_move(ctx, a, &refs.v[i], refs.v[i].src, tgt, drop);
}
return moved;
}
/* P4.2b: a repository built from more than one Dockerfile (otel builds
* every service as ghcr.io/open-telemetry/demo:<version>-<service>) is no
* single image. Each component of its build tags built from one Dockerfile
* becomes an Image of its own, which its build edges leave from; every
* reference (K8s container, compose service, FROM) whose tag names a
* component uses that Image; all else stays on the repository Image.
* Recomputed whole each run from the edges in the buffer. */
static int dk_image_hubs(cbm_pipeline_ctx_t *ctx, CBMArena *a) {
const cbm_gbuf_node_t **nodes = NULL;
int n = 0;
if (cbm_gbuf_find_by_label(ctx->gbuf, "Image", &nodes, &n) != 0 || n == 0) {
return 0;
}
dk_img_t *imgs = cbm_arena_alloc(a, (size_t)n * sizeof(*imgs));
if (!imgs) {
return 0;
}
for (int i = 0; i < n; i++) {
char repo[CBM_SZ_1K];
dk_repo_of(nodes[i]->name, repo, sizeof(repo));
imgs[i] = (dk_img_t){nodes[i]->id, cbm_arena_strdup(a, nodes[i]->name),
cbm_arena_strdup(a, repo)};
}
qsort(imgs, (size_t)n, sizeof(*imgs), dk_img_cmp);
dk_drop_t drop = {0};
int moved = 0;
for (int g = 0; g < n;) {
int h = g + SKIP_ONE;
while (h < n && strcmp(imgs[h].repo, imgs[g].repo) == 0) {
h++;
}
moved += dk_hub_repo(ctx, a, imgs + g, h - g, &drop);
g = h;
}
cbm_gbuf_delete_edges(ctx->gbuf, drop.v, drop.n);
return moved;
}
void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
int file_count) {
TSParser *parser = NULL;
@@ -1531,6 +1964,7 @@ void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fi
cbm_ht_free(ix.gomain);
}
edges += dk_link_builds(ctx, &arena);
edges += dk_image_hubs(ctx, &arena);
cbm_arena_destroy(&arena);
if (edges > 0) {
char buf[CBM_SZ_32];
+3
View File
@@ -3607,6 +3607,9 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
if (check_cancel(p)) {
return CBM_NOT_FOUND;
}
/* A repository Image whose edges all went to its components (pass_docker.c
* P4.2b) is left without any: it goes, as on the incremental routes. */
(void)cbm_gbuf_delete_unreferenced(p->gbuf, "Image", DK_IMAGE_EDGES);
return rc;
}
+21 -6
View File
@@ -283,18 +283,16 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c
return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND;
}
int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a,
const char ***out) {
/* The first column of `sql` (?1 = the project) as arena strings in *out. */
static int delta_query_strings(cbm_store_t *store, const char *project, const char *sql,
CBMArena *a, const char ***out) {
*out = NULL;
sqlite3 *db = cbm_store_get_db(store);
if (!db) {
return CBM_NOT_FOUND;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db,
"SELECT file_path FROM nodes WHERE project = ?1 AND label = 'File'"
" AND json_extract(properties, '$.declares_build') = 1",
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
return CBM_NOT_FOUND;
}
sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
@@ -325,6 +323,23 @@ int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena
return n;
}
int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a,
const char ***out) {
return delta_query_strings(store, project,
"SELECT file_path FROM nodes WHERE project = ?1 AND label = 'File'"
" AND json_extract(properties, '$.declares_build') = 1",
a, out);
}
int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena *a,
const char ***out) {
return delta_query_strings(store, project,
"SELECT qualified_name FROM nodes WHERE project = ?1 AND"
" label = 'Image' AND json_extract(properties, '$.component')"
" IS NOT NULL ORDER BY qualified_name",
a, out);
}
int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type,
const char *label) {
sqlite3 *db = cbm_store_get_db(store);
+52
View File
@@ -1164,6 +1164,46 @@ static bool incr_edge_type_is_recomputed(const char *type) {
strcmp(type, "FILE_CHANGES_WITH") == 0 || strcmp(type, "DATA_FLOWS") == 0);
}
static int incr_strcmp(const void *x, const void *y) {
return strcmp(*(const char *const *)x, *(const char *const *)y);
}
/* Are the component Images this run built (pass_docker.c P4.2b: an Image
* with a `component` property and a build edge in `gb`) exactly `before`
* (sorted QNs)? */
static bool incr_image_components_equal(const cbm_gbuf_t *gb, const char *const *before, int nb,
CBMArena *a) {
const cbm_gbuf_node_t **nodes = NULL;
int n = 0;
if (cbm_gbuf_find_by_label(gb, "Image", &nodes, &n) != 0 || n == 0) {
return nb == 0;
}
const char **now = cbm_arena_alloc(a, (size_t)n * sizeof(*now));
if (!now) {
return false;
}
int k = 0;
for (int i = 0; i < n; i++) {
const cbm_gbuf_edge_t **out = NULL;
int ne = 0;
if (nodes[i]->properties_json && strstr(nodes[i]->properties_json, "\"component\":") &&
cbm_gbuf_find_edges_by_source_type(gb, nodes[i]->id, "DEPENDS_ON", &out, &ne) == 0 &&
ne > 0) {
now[k++] = nodes[i]->qualified_name;
}
}
if (k != nb) {
return false;
}
qsort((void *)now, (size_t)k, sizeof(*now), incr_strcmp);
for (int i = 0; i < k; i++) {
if (strcmp(now[i], before[i]) != 0) {
return false;
}
}
return true;
}
/* The files whose File node the shell pass marked as running a container
* build (pass_shell.c), from the full-load buffer, into arena `a`; returns
* how many. */
@@ -2898,6 +2938,11 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
/* before the purge drops any of their edges */
decl_count = cbm_delta_build_decl_files(staging, project, &decl_arena, &decl_files);
decl_count = decl_count > 0 ? decl_count : 0;
const char **hub_images = NULL;
int hub_count = cbm_delta_image_components(staging, project, &decl_arena, &hub_images);
if (hub_count < 0) {
goto out;
}
/* Purge set = closure files + deleted files. */
int purge_count = ci + deleted_count;
@@ -3127,6 +3172,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
}
goto out;
}
/* A repository split differently (pass_docker.c P4.2b) leaves the
* references of files not repaired here on the old component Images:
* only a full rebuild re-points them. */
if (!incr_image_components_equal(gbuf, hub_images, hub_count, &decl_arena)) {
cbm_log_info("delta.fallback", "reason", "image_components_changed");
goto out;
}
cbm_log_info("delta.repair", "files", itoa_buf(ci), "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
/* Doc-link rows: the previous rows of every file not re-extracted (or
+8
View File
@@ -1383,6 +1383,14 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c
int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a,
const char ***out);
/* The QNs of the component Images a split repository gave its components
* (pass_docker.c P4.2b: Images with a `component` property), sorted, into
* arena `a`; returns how many, or -1 on failure. A closure repair compares
* them with the run's own: references in files it does not repair point at
* the old ones. */
int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena *a,
const char ***out);
/* Delete the project's `type` edges whose source node has label `label`;
* returns how many, or -1 on failure. */
int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type,
+116
View File
@@ -2205,6 +2205,121 @@ TEST(probe_shell_docker_builds_build_images) {
PASS();
}
/* The integer `sql` (?1 = the project) yields on the indexed files; -1 on
* failure. */
static int gpf_query_int(const GpfFile *files, int nfiles, const char *sql) {
GpfProj lp;
cbm_store_t *store = gpf_index_files(&lp, files, nfiles);
int out = -1;
if (store) {
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC);
if (sqlite3_step(st) == SQLITE_ROW) {
out = sqlite3_column_int(st, 0);
}
}
sqlite3_finalize(st);
}
gpf_cleanup(&lp, store);
return out;
}
/* A repository built from several Dockerfiles, one per component tag (otel's
* demo:<version>-<service>), is no single image: each component built from
* one Dockerfile is an Image of its own (`<repo>:*-<component>`), and a
* reference whose tag names a component uses it -- whatever its version. A
* tag naming none, a component built from two Dockerfiles, and a repository
* built from one Dockerfile stay on the repository Image. */
TEST(probe_image_hub_splits_by_component) {
static const GpfFile files[] = {
{"svc/a/Dockerfile", "FROM alpine:3.20\n"},
{"svc/b/Dockerfile", "FROM alpine:3.20\n"},
{"svc/c/Dockerfile", "FROM alpine:3.20\n"},
{"solo/Dockerfile", "FROM alpine:3.20\n"},
{"tool/Dockerfile", "FROM alpine:3.20\n"},
{"tool/Dockerfile.test", "FROM alpine:3.20\n"},
{"w1/Dockerfile", "FROM alpine:3.20\n"},
{"w2/Dockerfile.alt", "FROM alpine:3.20\n"},
{"v/Dockerfile", "FROM alpine:3.20\n"},
{"p/Dockerfile", "FROM alpine:3.20\n"},
{"q/Dockerfile", "FROM alpine:3.20\n"},
{".env", "V=1.0\n"},
{"compose.yaml",
"services:\n a:\n image: acme/demo:${V}-a\n build: ./svc/a\n"
" b:\n image: acme/demo:${V}-b\n build: ./svc/b\n"
" c:\n image: acme/demo:${V}-c\n build: ./svc/c\n"
" solo:\n image: acme/solo:1\n build: ./solo\n"
" tool:\n image: acme/tool\n build: ./tool\n"
" tooltest:\n image: acme/tool:test\n build:\n context: ./tool\n"
" dockerfile: Dockerfile.test\n"
" w1:\n image: acme/x:1-w\n build: ./w1\n"
" w2:\n image: acme/x:1-w\n build:\n context: ./w2\n"
" dockerfile: Dockerfile.alt\n"
" v:\n image: acme/x:1-v\n build: ./v\n"
" p:\n image: acme/y:1-p\n build: ./p\n"
" q:\n image: acme/y:1-q\n build: ./q\n"},
{"k8s.yaml",
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: b\nspec:\n template:\n"
" spec:\n containers:\n - name: b\n image: acme/demo:0.9-b\n"
"---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: z\nspec:\n"
" template:\n spec:\n containers:\n - name: z\n"
" image: acme/demo:0.9-zzz\n"
"---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: s\nspec:\n"
" template:\n spec:\n containers:\n - name: s\n"
" image: acme/solo:2\n"}};
enum { NF = sizeof(files) / sizeof(files[0]) };
char *got = gpf_builds(files, NF);
static const char want_builds[] = "docker.io/acme/demo:*-a -> svc/a/Dockerfile (compose)\n"
"docker.io/acme/demo:*-b -> svc/b/Dockerfile (compose)\n"
"docker.io/acme/demo:*-c -> svc/c/Dockerfile (compose)\n"
"docker.io/acme/solo -> solo/Dockerfile (compose)\n"
"docker.io/acme/tool -> tool/Dockerfile (compose)\n"
"docker.io/acme/tool:test -> tool/Dockerfile.test (compose)\n"
"docker.io/acme/x -> w1/Dockerfile (compose)\n"
"docker.io/acme/x -> w2/Dockerfile.alt (compose)\n"
"docker.io/acme/x:*-v -> v/Dockerfile (compose)\n"
"docker.io/acme/y:*-p -> p/Dockerfile (compose)\n"
"docker.io/acme/y:*-q -> q/Dockerfile (compose)";
if (!got || strcmp(got, want_builds) != 0) {
fprintf(stderr, "builds got:\n%s\nwant:\n%s\n", got ? got : "(null)", want_builds);
free(got);
FAIL("component Images' build edges");
}
free(got);
got = gpf_image_users(files, NF);
static const char want_users[] = "Deployment/b -> docker.io/acme/demo:*-b\n"
"Deployment/s -> docker.io/acme/solo\n"
"Deployment/z -> docker.io/acme/demo\n"
"Service/a -> docker.io/acme/demo:*-a\n"
"Service/b -> docker.io/acme/demo:*-b\n"
"Service/c -> docker.io/acme/demo:*-c\n"
"Service/p -> docker.io/acme/y:*-p\n"
"Service/q -> docker.io/acme/y:*-q\n"
"Service/solo -> docker.io/acme/solo\n"
"Service/tool -> docker.io/acme/tool\n"
"Service/tooltest -> docker.io/acme/tool:test\n"
"Service/v -> docker.io/acme/x:*-v\n"
"Service/w1 -> docker.io/acme/x\n"
"Service/w2 -> docker.io/acme/x";
if (!got || strcmp(got, want_users) != 0) {
fprintf(stderr, "image users got:\n%s\nwant:\n%s\n", got ? got : "(null)", want_users);
free(got);
FAIL("references on component Images");
}
free(got);
/* acme/y's repository Image lost every edge to its components: none left */
int orphans = gpf_query_int(files, NF,
"SELECT COUNT(*) FROM nodes n WHERE n.project=?1 AND "
"n.label='Image' AND NOT EXISTS (SELECT 1 FROM edges e WHERE "
"e.source_id=n.id OR e.target_id=n.id)");
if (orphans != 0) {
fprintf(stderr, "Images without an edge: %d\n", orphans);
FAIL("a repository Image left without edges");
}
PASS();
}
/* Every CONFIGURES {strategy:env_set} edge as
* "<owner> -> <container | ->/<name> (<source>)"; then the joins a scoped
* EnvVar must not have: "#PORT=<EnvVar nodes named PORT>" and
@@ -4573,6 +4688,7 @@ SUITE(grammar_probe_f) {
RUN_TEST(probe_compose_services_are_resources);
RUN_TEST(probe_shell_docker_builds_build_images);
RUN_TEST(probe_container_env_sets_scoped_envvars);
RUN_TEST(probe_image_hub_splits_by_component);
RUN_TEST(probe_gitops_objects_deploy_their_units);
RUN_TEST(probe_shell_kubectl_deploys_what_it_applies);
RUN_TEST(probe_shell_helm_deploys_its_chart);
+68
View File
@@ -3746,6 +3746,73 @@ TEST(incremental_env_edit_replaces_scoped_envvars) {
PASS();
}
/* A repository split by component (pass_docker.c P4.2b). `rename`: component
* b becomes bee -- the split changes, and the reference in compose.run.yaml,
* a file this run does not repair, must leave the old Image `*-b`: a closure
* repair cannot re-point it, so the run rebuilds. Else an edit keeping the
* split: a closure repair. Returns "repo=N b=M bimg=K" (runner's references
* to the repository Image / to `*-b`, `*-b` Images), or NULL. */
static char *incremental_image_split(bool rename, cbm_incremental_route_t *route) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_imagesplit_XXXXXX");
if (!cbm_mkdtemp(tmp)) {
return NULL;
}
write_temp_file(tmp, "svc/a/Dockerfile", "FROM alpine:3.20\n");
write_temp_file(tmp, "svc/b/Dockerfile", "FROM alpine:3.20\n");
write_temp_file(tmp, "compose.yaml",
"services:\n a:\n image: acme/demo:1-a\n build: ./svc/a\n"
" b:\n image: acme/demo:1-b\n build: ./svc/b\n");
write_temp_file(tmp, "compose.run.yaml", "services:\n runner:\n image: acme/demo:2-b\n");
char db_path[512];
snprintf(db_path, sizeof(db_path), "%s/imagesplit.db", tmp);
static const char to_repo[] =
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.type='USAGE' AND s.name='Service/runner' AND "
"t.name='docker.io/acme/demo'";
static const char to_b[] =
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.type='USAGE' AND s.name='Service/runner' AND "
"t.name='docker.io/acme/demo:*-b'";
static const char b_img[] =
"SELECT COUNT(*) FROM nodes WHERE label='Image' AND name='docker.io/acme/demo:*-b'";
if (k8s_inc_index(tmp, db_path) != 0 || k8s_inc_count(db_path, to_b) != 1) {
th_rmtree(tmp);
return NULL;
}
write_temp_file(tmp, "compose.yaml",
rename ? "services:\n a:\n image: acme/demo:1-a\n build: ./svc/a\n"
" b:\n image: acme/demo:1-bee\n build: ./svc/b\n"
: "services:\n a:\n image: acme/demo:1-a\n build: ./svc/a\n"
" environment:\n - MODE\n"
" b:\n image: acme/demo:1-b\n build: ./svc/b\n");
char *out = NULL;
if (k8s_inc_index(tmp, db_path) == 0) {
char buf[64];
snprintf(buf, sizeof(buf), "repo=%d b=%d bimg=%d", k8s_inc_count(db_path, to_repo),
k8s_inc_count(db_path, to_b), k8s_inc_count(db_path, b_img));
out = strdup(buf);
}
*route = cbm_pipeline_incremental_test_last_route();
th_rmtree(tmp);
return out;
}
TEST(incremental_image_split_change_rebuilds) {
cbm_incremental_route_t route = CBM_INCREMENTAL_ROUTE_NONE;
char *got = incremental_image_split(true, &route);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_FORCED_FULL);
ASSERT_NOT_NULL(got);
ASSERT_STR_EQ(got, "repo=1 b=0 bimg=0");
free(got);
got = incremental_image_split(false, &route);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
ASSERT_NOT_NULL(got);
ASSERT_STR_EQ(got, "repo=0 b=1 bimg=1");
free(got);
PASS();
}
/* A compose project's .env feeds its compose file (variables in image,
* build): changing it is no closure repair -- the unchanged compose file's
* services would keep the old image. It rebuilds; no stale image remains. */
@@ -19513,6 +19580,7 @@ SUITE(pipeline_semantic_manifest_repro) {
RUN_TEST(incremental_dockerfile_edit_keeps_build_edges);
RUN_TEST(incremental_compose_edit_replaces_build_edges);
RUN_TEST(incremental_env_edit_replaces_scoped_envvars);
RUN_TEST(incremental_image_split_change_rebuilds);
RUN_TEST(incremental_compose_dotenv_change_rebuilds);
RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges);
RUN_TEST(incremental_second_build_declarer_keeps_the_edge);