diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 03a3bc0dd..a988fa025 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -1919,6 +1919,34 @@ int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const cha return gbuf_delete_edges_if(gb, gbuf_edge_from_label, &tl); } +static int gbuf_ptr_cmp(const void *k, const void *v) { + uintptr_t a = (uintptr_t)*(const cbm_gbuf_edge_t *const *)k; + uintptr_t b = (uintptr_t)*(const cbm_gbuf_edge_t *const *)v; + return (a > b) - (a < b); +} + +typedef struct { + const cbm_gbuf_edge_t **edges; + int n; +} gbuf_edge_set_t; + +static bool gbuf_edge_in_set(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) { + (void)gb; + const gbuf_edge_set_t *s = ud; + return bsearch(&e, s->edges, (size_t)s->n, sizeof(*s->edges), gbuf_ptr_cmp) != NULL; +} + +/* By the edges themselves, not their ids: after the parallel resolve merge, + * cbm_gbuf_set_next_id can hand an id out twice. */ +int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n) { + if (!gb || !edges || n <= 0) { + return 0; + } + qsort((void *)edges, (size_t)n, sizeof(*edges), gbuf_ptr_cmp); + gbuf_edge_set_t s = {edges, n}; + return gbuf_delete_edges_if(gb, gbuf_edge_in_set, &s); +} + /* ── Merge ───────────────────────────────────────────────────────── */ /* Free remap hash table entries (key = heap string, value = heap int64_t*) */ diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 66c3377c1..f2fdaa59a 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -162,6 +162,10 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type); /* Delete the edges of `type` whose source node has label `label`. */ int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const char *label); +/* Delete edges[0..n) (pointers found in this buffer; the array is sorted in + * place). */ +int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n); + /* ── Vector storage (for semantic embeddings) ───────────────────── */ /* Store an int8-quantized vector for a node. The vector data is copied. diff --git a/src/pipeline/pass_docker.c b/src/pipeline/pass_docker.c index 85aa08d3c..fb6d8730b 100644 --- a/src/pipeline/pass_docker.c +++ b/src/pipeline/pass_docker.c @@ -54,6 +54,7 @@ #include "lang_specs.h" #include +#include #include #include #include @@ -1465,6 +1466,438 @@ static void dk_index_build(cbm_pipeline_ctx_t *ctx, CBMArena *a, dk_index_t *ix) } } +/* ── Image tag sub-identity (P4.2b) ─────────────────────────────── */ + +_Atomic uint64_t g_dk_hub_edges = 0; /* complexity guard: Image edges a run examines */ + +/* A plain JSON string property (an image reference holds no escapes). */ +static bool dk_prop(const char *props, const char *key, char *out, size_t out_sz) { + char pat[CBM_SZ_64]; + snprintf(pat, sizeof(pat), "\"%s\":\"", key); + const char *p = props ? strstr(props, pat) : NULL; + if (!p) { + return false; + } + p += strlen(pat); + const char *e = strchr(p, '"'); + if (!e || (size_t)(e - p) >= out_sz || memchr(p, '\\', (size_t)(e - p))) { + return false; + } + memcpy(out, p, (size_t)(e - p)); + out[e - p] = '\0'; + return true; +} + +/* The tag of image reference `ref`: after the ':' that follows its last + * '/' (a registry port is no tag), up to a '@' digest; "" for none. */ +static void dk_tag(const char *ref, char *out, size_t sz) { + out[0] = '\0'; + const char *slash = strrchr(ref, '/'); + const char *colon = strchr(slash ? slash : ref, ':'); + if (colon) { + snprintf(out, sz, "%.*s", (int)strcspn(colon + SKIP_ONE, "@"), colon + SKIP_ONE); + } +} + +/* The repository of an Image node's name ("" or ":"). */ +static void dk_repo_of(const char *name, char *out, size_t sz) { + const char *slash = strrchr(name, '/'); + const char *colon = strchr(slash ? slash : name, ':'); + snprintf(out, sz, "%.*s", (int)(colon ? (size_t)(colon - name) : strlen(name)), name); +} + +static bool dk_sep(char c) { + return c == '-' || c == '_' || c == '.'; +} + +/* Does `comp` occur in `tag` as whole separator-delimited parts? */ +static bool dk_tag_has(const char *tag, const char *comp) { + size_t cl = strlen(comp); + for (const char *p = strstr(tag, comp); p; p = strstr(p + SKIP_ONE, comp)) { + if ((p == tag || dk_sep(p[-SKIP_ONE])) && (p[cl] == '\0' || dk_sep(p[cl]))) { + return true; + } + } + return false; +} + +/* An edge of a repository's Images, copied before any edge moves. */ +typedef struct { + const cbm_gbuf_edge_t *edge; /* stable until deleted */ + int64_t src; + int64_t tgt; + const char *type; + const char *props; + const char *tag; + int comp; /* its component; -1: none */ +} dk_hub_edge_t; + +typedef struct { + dk_hub_edge_t *v; + int n; + int cap; +} dk_hub_edges_t; + +typedef struct { + const cbm_gbuf_edge_t **v; + int n; + int cap; +} dk_drop_t; + +typedef struct { + int64_t id; + const char *name; + const char *repo; +} dk_img_t; + +static int dk_img_cmp(const void *x, const void *y) { + const dk_img_t *a = x; + const dk_img_t *b = y; + int c = strcmp(a->repo, b->repo); + return c ? c : strcmp(a->name, b->name); +} + +/* Grow an arena array of `n` items of `size` to hold one more. */ +static void *dk_grow(CBMArena *a, void *v, int n, int *cap, size_t size) { + if (n < *cap) { + return v; + } + int nc = *cap ? *cap * PAIR_LEN : CBM_SZ_16; + void *g = cbm_arena_alloc(a, (size_t)nc * size); + if (g && n) { + memcpy(g, v, (size_t)n * size); + } + if (g) { + *cap = nc; + } + return g; +} + +static void dk_hub_push(CBMArena *a, dk_hub_edges_t *l, const cbm_gbuf_edge_t *e, const char *tag) { + dk_hub_edge_t *v = dk_grow(a, l->v, l->n, &l->cap, sizeof(*v)); + if (!v) { + return; + } + l->v = v; + l->v[l->n++] = (dk_hub_edge_t){e, + e->source_id, + e->target_id, + cbm_arena_strdup(a, e->type), + cbm_arena_strdup(a, e->properties_json), + cbm_arena_strdup(a, tag), + -1}; +} + +/* Is an edge of `type` with properties `p` a reference to an image -- + * USAGE {ref:image}, or DEPENDS_ON {via:from|copy-from}? */ +static bool dk_hub_ref(const char *type, const char *p) { + char v[CBM_SZ_32]; + if (strcmp(type, "USAGE") == 0) { + return dk_prop(p, "ref", v, sizeof(v)) && strcmp(v, "image") == 0; + } + return dk_prop(p, "via", v, sizeof(v)) && + (strcmp(v, "from") == 0 || strcmp(v, "copy-from") == 0); +} + +/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}) and + * references in, each with the tag of its `image` reference. */ +static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, + dk_hub_edges_t *builds, dk_hub_edges_t *refs) { + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + char ref[CBM_SZ_1K]; + char tag[CBM_SZ_256]; + if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, img, "DEPENDS_ON", &es, &n) == 0) { + for (int i = 0; i < n; i++) { + char via[CBM_SZ_32]; + if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via)) || + strcmp(via, "build") != 0) { + continue; + } + tag[0] = '\0'; + if (dk_prop(es[i]->properties_json, "image", ref, sizeof(ref))) { + dk_tag(ref, tag, sizeof(tag)); + } + dk_hub_push(a, builds, es[i], tag); + } + atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed); + } + static const char *const IN[] = {"USAGE", "DEPENDS_ON"}; + for (size_t t = 0; t < sizeof(IN) / sizeof(IN[0]); t++) { + if (cbm_gbuf_find_edges_by_target_type(ctx->gbuf, img, IN[t], &es, &n) != 0) { + continue; + } + atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed); + for (int i = 0; i < n; i++) { + if (dk_hub_ref(IN[t], es[i]->properties_json) && + dk_prop(es[i]->properties_json, "image", ref, sizeof(ref))) { + dk_tag(ref, tag, sizeof(tag)); + dk_hub_push(a, refs, es[i], tag); + } + } + } +} + +/* The number of distinct Dockerfiles the build edges target. */ +static int dk_hub_dockerfiles(CBMArena *a, const dk_hub_edges_t *b) { + CBMHashTable *seen = cbm_ht_create(0); + int n = 0; + for (int i = 0; seen && i < b->n; i++) { + const char *k = cbm_arena_sprintf(a, "%lld", (long long)b->v[i].tgt); + if (k && !cbm_ht_has(seen, k)) { + cbm_ht_set(seen, k, (void *)SKIP_ONE); + n++; + } + } + if (seen) { + cbm_ht_free(seen); + } + return n; +} + +/* The common prefix / suffix of the build tags, cut back to a separator and + * leaving every tag a non-empty middle where it can: *pre, *suf. */ +static void dk_hub_affixes(const dk_hub_edges_t *b, size_t *pre, size_t *suf) { + const char *t0 = b->v[0].tag; + size_t l0 = strlen(t0); + size_t p = l0; + size_t s = l0; + for (int i = SKIP_ONE; i < b->n; i++) { + const char *t = b->v[i].tag; + size_t tl = strlen(t); + size_t k = 0; + while (k < p && k < tl && t[k] == t0[k]) { + k++; + } + p = k; + k = 0; + while (k < s && k < tl && t[tl - SKIP_ONE - k] == t0[l0 - SKIP_ONE - k]) { + k++; + } + s = k; + } + while (p > 0 && !dk_sep(t0[p - SKIP_ONE])) { + p--; + } + while (s > 0 && !dk_sep(t0[l0 - s])) { + s--; + } + for (int i = 0; i < b->n; i++) { + size_t tl = strlen(b->v[i].tag); + while (s > 0 && p + s >= tl) { + for (s--; s > 0 && !dk_sep(t0[l0 - s]); s--) {} + } + while (p > 0 && p + s >= tl) { + for (p--; p > 0 && !dk_sep(t0[p - SKIP_ONE]); p--) {} + } + } + *pre = p; + *suf = s; +} + +/* Components of a repository built from several Dockerfiles: each build + * tag without the tags' common prefix / suffix (otel's latest-frontend, + * latest-cart -> frontend, cart). A component built from exactly one + * Dockerfile, holding no unknown part, is kept (comps[k] non-NULL); each + * build edge's comp is set. Returns the number of component slots (0: + * none kept). */ +static int dk_hub_components(CBMArena *a, dk_hub_edges_t *b, size_t pre, size_t suf, + const char ***comps) { + const char **names = cbm_arena_alloc(a, (size_t)b->n * sizeof(*names)); + int64_t *dfs = cbm_arena_alloc(a, (size_t)b->n * sizeof(*dfs)); + CBMHashTable *index = cbm_ht_create(0); /* component -> slot + 1 */ + int nc = 0; + for (int i = 0; names && dfs && index && i < b->n; i++) { + const char *t = b->v[i].tag; + size_t tl = strlen(t); + const char *c = tl > pre + suf ? cbm_arena_strndup(a, t + pre, tl - pre - suf) : NULL; + if (!c || strpbrk(c, "$*{}()`\"\\")) { + continue; + } + int k = (int)(intptr_t)cbm_ht_get(index, c) - SKIP_ONE; + if (k < 0) { + k = nc++; + names[k] = c; + dfs[k] = b->v[i].tgt; + cbm_ht_set(index, c, (void *)(intptr_t)(k + SKIP_ONE)); + } else if (names[k] && dfs[k] != b->v[i].tgt) { + names[k] = NULL; /* one name, two Dockerfiles: no identity */ + } + b->v[i].comp = k; + } + if (index) { + cbm_ht_free(index); + } + if (!names || !dfs) { + return 0; + } + int kept = 0; + for (int k = 0; k < nc; k++) { + kept += names[k] ? SKIP_ONE : 0; + } + for (int i = 0; i < b->n; i++) { + if (b->v[i].comp >= 0 && !names[b->v[i].comp]) { + b->v[i].comp = -1; + } + } + *comps = names; + return kept ? nc : 0; +} + +/* The component a reference's tag names: the longest kept one occurring in + * it as whole parts; -1 for none, or for two of that length. */ +static int dk_hub_match(const char *tag, const char *const *comps, int nc) { + int best = -1; + size_t bl = 0; + bool tie = false; + for (int k = 0; k < nc; k++) { + if (!comps[k] || !dk_tag_has(tag, comps[k])) { + continue; + } + size_t l = strlen(comps[k]); + if (l > bl) { + best = k; + bl = l; + tie = false; + } else if (l == bl) { + tie = true; + } + } + return tie ? -1 : best; +} + +/* The Image of a component: `:` (`*-frontend`: the + * prefix's version masked) {repository, component, tag_pattern}. */ +static int64_t dk_hub_image(cbm_pipeline_ctx_t *ctx, const char *repo, const char *comp, + char pre_sep, char suf_sep) { + char head[PAIR_LEN + SKIP_ONE] = {0}; + char tail[PAIR_LEN + SKIP_ONE] = {0}; + if (pre_sep) { + head[0] = '*'; + head[SKIP_ONE] = pre_sep; + } + if (suf_sep) { + tail[0] = suf_sep; + tail[SKIP_ONE] = '*'; + } + char pattern[CBM_SZ_512]; + char name[CBM_SZ_1K]; + char qn[CBM_SZ_2K]; + snprintf(pattern, sizeof(pattern), "%s%s%s", head, comp, tail); + snprintf(name, sizeof(name), "%s:%s", repo, pattern); + snprintf(qn, sizeof(qn), "__image__%s", name); + char esc_r[CBM_SZ_1K]; + char esc_c[CBM_SZ_1K]; + char esc_p[CBM_SZ_1K]; + cbm_json_escape(esc_r, (int)sizeof(esc_r), repo); + cbm_json_escape(esc_c, (int)sizeof(esc_c), comp); + cbm_json_escape(esc_p, (int)sizeof(esc_p), pattern); + char props[CBM_SZ_4K]; + snprintf(props, sizeof(props), + "{\"repository\":\"%s\",\"component\":\"%s\",\"tag_pattern\":\"%s\"}", esc_r, esc_c, + esc_p); + return cbm_gbuf_upsert_node(ctx->gbuf, "Image", name, qn, "", 0, 0, props); +} + +/* Move edge `e` to (src, tgt): insert the copy, queue `e` for deletion. */ +static int dk_hub_move(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_hub_edge_t *e, int64_t src, + int64_t tgt, dk_drop_t *drop) { + if ((src == e->src && tgt == e->tgt) || src <= 0 || tgt <= 0) { + return 0; + } + const cbm_gbuf_edge_t **v = dk_grow(a, (void *)drop->v, drop->n, &drop->cap, sizeof(*v)); + if (!v) { + return 0; + } + drop->v = v; + drop->v[drop->n++] = e->edge; + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, e->type, e->props); + return SKIP_ONE; +} + +/* One repository's Images (imgs[0..n)): split it by component when it is + * built from several Dockerfiles, else keep (or bring back) everything on + * the repository Image. Returns the edges moved. */ +static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *imgs, int n, + dk_drop_t *drop) { + dk_hub_edges_t builds = {0}; + dk_hub_edges_t refs = {0}; + bool has_sub = false; + for (int i = 0; i < n; i++) { + has_sub = has_sub || strcmp(imgs[i].name, imgs[i].repo) != 0; + dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs); + } + const char **comps = NULL; + size_t pre = 0; + size_t suf = 0; + int nc = 0; + if (builds.n > SKIP_ONE && dk_hub_dockerfiles(a, &builds) > SKIP_ONE) { + dk_hub_affixes(&builds, &pre, &suf); + nc = dk_hub_components(a, &builds, pre, suf, &comps); + } + if (nc == 0 && !has_sub) { + return 0; + } + int64_t repo_img = cbm_pipeline_image_node(ctx, imgs[0].repo); + int64_t *sub = nc ? cbm_arena_alloc(a, (size_t)nc * sizeof(*sub)) : NULL; + const char *t0 = builds.n ? builds.v[0].tag : ""; + char pre_sep = pre ? t0[pre - SKIP_ONE] : '\0'; + char suf_sep = suf ? t0[strlen(t0) - suf] : '\0'; + for (int k = 0; k < nc && sub; k++) { + sub[k] = comps[k] ? dk_hub_image(ctx, imgs[0].repo, comps[k], pre_sep, suf_sep) : 0; + } + int moved = 0; + for (int i = 0; i < builds.n; i++) { + int k = builds.v[i].comp; + int64_t src = sub && k >= 0 && sub[k] > 0 ? sub[k] : repo_img; + moved += dk_hub_move(ctx, a, &builds.v[i], src, builds.v[i].tgt, drop); + } + for (int i = 0; i < refs.n; i++) { + int k = sub ? dk_hub_match(refs.v[i].tag, comps, nc) : -1; + int64_t tgt = k >= 0 && sub[k] > 0 ? sub[k] : repo_img; + moved += dk_hub_move(ctx, a, &refs.v[i], refs.v[i].src, tgt, drop); + } + return moved; +} + +/* P4.2b: a repository built from more than one Dockerfile (otel builds + * every service as ghcr.io/open-telemetry/demo:-) is no + * single image. Each component of its build tags built from one Dockerfile + * becomes an Image of its own, which its build edges leave from; every + * reference (K8s container, compose service, FROM) whose tag names a + * component uses that Image; all else stays on the repository Image. + * Recomputed whole each run from the edges in the buffer. */ +static int dk_image_hubs(cbm_pipeline_ctx_t *ctx, CBMArena *a) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Image", &nodes, &n) != 0 || n == 0) { + return 0; + } + dk_img_t *imgs = cbm_arena_alloc(a, (size_t)n * sizeof(*imgs)); + if (!imgs) { + return 0; + } + for (int i = 0; i < n; i++) { + char repo[CBM_SZ_1K]; + dk_repo_of(nodes[i]->name, repo, sizeof(repo)); + imgs[i] = (dk_img_t){nodes[i]->id, cbm_arena_strdup(a, nodes[i]->name), + cbm_arena_strdup(a, repo)}; + } + qsort(imgs, (size_t)n, sizeof(*imgs), dk_img_cmp); + dk_drop_t drop = {0}; + int moved = 0; + for (int g = 0; g < n;) { + int h = g + SKIP_ONE; + while (h < n && strcmp(imgs[h].repo, imgs[g].repo) == 0) { + h++; + } + moved += dk_hub_repo(ctx, a, imgs + g, h - g, &drop); + g = h; + } + cbm_gbuf_delete_edges(ctx->gbuf, drop.v, drop.n); + return moved; +} + void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { TSParser *parser = NULL; @@ -1531,6 +1964,7 @@ void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fi cbm_ht_free(ix.gomain); } edges += dk_link_builds(ctx, &arena); + edges += dk_image_hubs(ctx, &arena); cbm_arena_destroy(&arena); if (edges > 0) { char buf[CBM_SZ_32]; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index fa966538a..95935d0fc 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -3607,6 +3607,9 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (check_cancel(p)) { return CBM_NOT_FOUND; } + /* A repository Image whose edges all went to its components (pass_docker.c + * P4.2b) is left without any: it goes, as on the incremental routes. */ + (void)cbm_gbuf_delete_unreferenced(p->gbuf, "Image", DK_IMAGE_EDGES); return rc; } diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index 9d6a1b86b..0ccbbce48 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -283,18 +283,16 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; } -int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a, - const char ***out) { +/* The first column of `sql` (?1 = the project) as arena strings in *out. */ +static int delta_query_strings(cbm_store_t *store, const char *project, const char *sql, + CBMArena *a, const char ***out) { *out = NULL; sqlite3 *db = cbm_store_get_db(store); if (!db) { return CBM_NOT_FOUND; } sqlite3_stmt *stmt = NULL; - if (sqlite3_prepare_v2(db, - "SELECT file_path FROM nodes WHERE project = ?1 AND label = 'File'" - " AND json_extract(properties, '$.declares_build') = 1", - CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { return CBM_NOT_FOUND; } sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); @@ -325,6 +323,23 @@ int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena return n; } +int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out) { + return delta_query_strings(store, project, + "SELECT file_path FROM nodes WHERE project = ?1 AND label = 'File'" + " AND json_extract(properties, '$.declares_build') = 1", + a, out); +} + +int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out) { + return delta_query_strings(store, project, + "SELECT qualified_name FROM nodes WHERE project = ?1 AND" + " label = 'Image' AND json_extract(properties, '$.component')" + " IS NOT NULL ORDER BY qualified_name", + a, out); +} + int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type, const char *label) { sqlite3 *db = cbm_store_get_db(store); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 104f8d7fa..977b5e677 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1164,6 +1164,46 @@ static bool incr_edge_type_is_recomputed(const char *type) { strcmp(type, "FILE_CHANGES_WITH") == 0 || strcmp(type, "DATA_FLOWS") == 0); } +static int incr_strcmp(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* Are the component Images this run built (pass_docker.c P4.2b: an Image + * with a `component` property and a build edge in `gb`) exactly `before` + * (sorted QNs)? */ +static bool incr_image_components_equal(const cbm_gbuf_t *gb, const char *const *before, int nb, + CBMArena *a) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(gb, "Image", &nodes, &n) != 0 || n == 0) { + return nb == 0; + } + const char **now = cbm_arena_alloc(a, (size_t)n * sizeof(*now)); + if (!now) { + return false; + } + int k = 0; + for (int i = 0; i < n; i++) { + const cbm_gbuf_edge_t **out = NULL; + int ne = 0; + if (nodes[i]->properties_json && strstr(nodes[i]->properties_json, "\"component\":") && + cbm_gbuf_find_edges_by_source_type(gb, nodes[i]->id, "DEPENDS_ON", &out, &ne) == 0 && + ne > 0) { + now[k++] = nodes[i]->qualified_name; + } + } + if (k != nb) { + return false; + } + qsort((void *)now, (size_t)k, sizeof(*now), incr_strcmp); + for (int i = 0; i < k; i++) { + if (strcmp(now[i], before[i]) != 0) { + return false; + } + } + return true; +} + /* The files whose File node the shell pass marked as running a container * build (pass_shell.c), from the full-load buffer, into arena `a`; returns * how many. */ @@ -2898,6 +2938,11 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char /* before the purge drops any of their edges */ decl_count = cbm_delta_build_decl_files(staging, project, &decl_arena, &decl_files); decl_count = decl_count > 0 ? decl_count : 0; + const char **hub_images = NULL; + int hub_count = cbm_delta_image_components(staging, project, &decl_arena, &hub_images); + if (hub_count < 0) { + goto out; + } /* Purge set = closure files + deleted files. */ int purge_count = ci + deleted_count; @@ -3127,6 +3172,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char } goto out; } + /* A repository split differently (pass_docker.c P4.2b) leaves the + * references of files not repaired here on the old component Images: + * only a full rebuild re-points them. */ + if (!incr_image_components_equal(gbuf, hub_images, hub_count, &decl_arena)) { + cbm_log_info("delta.fallback", "reason", "image_components_changed"); + goto out; + } cbm_log_info("delta.repair", "files", itoa_buf(ci), "elapsed_ms", itoa_buf((int)elapsed_ms(t))); /* Doc-link rows: the previous rows of every file not re-extracted (or diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 140ce7587..72a57bf44 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1383,6 +1383,14 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a, const char ***out); +/* The QNs of the component Images a split repository gave its components + * (pass_docker.c P4.2b: Images with a `component` property), sorted, into + * arena `a`; returns how many, or -1 on failure. A closure repair compares + * them with the run's own: references in files it does not repair point at + * the old ones. */ +int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out); + /* Delete the project's `type` edges whose source node has label `label`; * returns how many, or -1 on failure. */ int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type, diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index c32b752d8..d4d2099f1 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -2205,6 +2205,121 @@ TEST(probe_shell_docker_builds_build_images) { PASS(); } +/* The integer `sql` (?1 = the project) yields on the indexed files; -1 on + * failure. */ +static int gpf_query_int(const GpfFile *files, int nfiles, const char *sql) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int out = -1; + if (store) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A repository built from several Dockerfiles, one per component tag (otel's + * demo:-), is no single image: each component built from + * one Dockerfile is an Image of its own (`:*-`), and a + * reference whose tag names a component uses it -- whatever its version. A + * tag naming none, a component built from two Dockerfiles, and a repository + * built from one Dockerfile stay on the repository Image. */ +TEST(probe_image_hub_splits_by_component) { + static const GpfFile files[] = { + {"svc/a/Dockerfile", "FROM alpine:3.20\n"}, + {"svc/b/Dockerfile", "FROM alpine:3.20\n"}, + {"svc/c/Dockerfile", "FROM alpine:3.20\n"}, + {"solo/Dockerfile", "FROM alpine:3.20\n"}, + {"tool/Dockerfile", "FROM alpine:3.20\n"}, + {"tool/Dockerfile.test", "FROM alpine:3.20\n"}, + {"w1/Dockerfile", "FROM alpine:3.20\n"}, + {"w2/Dockerfile.alt", "FROM alpine:3.20\n"}, + {"v/Dockerfile", "FROM alpine:3.20\n"}, + {"p/Dockerfile", "FROM alpine:3.20\n"}, + {"q/Dockerfile", "FROM alpine:3.20\n"}, + {".env", "V=1.0\n"}, + {"compose.yaml", + "services:\n a:\n image: acme/demo:${V}-a\n build: ./svc/a\n" + " b:\n image: acme/demo:${V}-b\n build: ./svc/b\n" + " c:\n image: acme/demo:${V}-c\n build: ./svc/c\n" + " solo:\n image: acme/solo:1\n build: ./solo\n" + " tool:\n image: acme/tool\n build: ./tool\n" + " tooltest:\n image: acme/tool:test\n build:\n context: ./tool\n" + " dockerfile: Dockerfile.test\n" + " w1:\n image: acme/x:1-w\n build: ./w1\n" + " w2:\n image: acme/x:1-w\n build:\n context: ./w2\n" + " dockerfile: Dockerfile.alt\n" + " v:\n image: acme/x:1-v\n build: ./v\n" + " p:\n image: acme/y:1-p\n build: ./p\n" + " q:\n image: acme/y:1-q\n build: ./q\n"}, + {"k8s.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: b\nspec:\n template:\n" + " spec:\n containers:\n - name: b\n image: acme/demo:0.9-b\n" + "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: z\nspec:\n" + " template:\n spec:\n containers:\n - name: z\n" + " image: acme/demo:0.9-zzz\n" + "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: s\nspec:\n" + " template:\n spec:\n containers:\n - name: s\n" + " image: acme/solo:2\n"}}; + enum { NF = sizeof(files) / sizeof(files[0]) }; + char *got = gpf_builds(files, NF); + static const char want_builds[] = "docker.io/acme/demo:*-a -> svc/a/Dockerfile (compose)\n" + "docker.io/acme/demo:*-b -> svc/b/Dockerfile (compose)\n" + "docker.io/acme/demo:*-c -> svc/c/Dockerfile (compose)\n" + "docker.io/acme/solo -> solo/Dockerfile (compose)\n" + "docker.io/acme/tool -> tool/Dockerfile (compose)\n" + "docker.io/acme/tool:test -> tool/Dockerfile.test (compose)\n" + "docker.io/acme/x -> w1/Dockerfile (compose)\n" + "docker.io/acme/x -> w2/Dockerfile.alt (compose)\n" + "docker.io/acme/x:*-v -> v/Dockerfile (compose)\n" + "docker.io/acme/y:*-p -> p/Dockerfile (compose)\n" + "docker.io/acme/y:*-q -> q/Dockerfile (compose)"; + if (!got || strcmp(got, want_builds) != 0) { + fprintf(stderr, "builds got:\n%s\nwant:\n%s\n", got ? got : "(null)", want_builds); + free(got); + FAIL("component Images' build edges"); + } + free(got); + got = gpf_image_users(files, NF); + static const char want_users[] = "Deployment/b -> docker.io/acme/demo:*-b\n" + "Deployment/s -> docker.io/acme/solo\n" + "Deployment/z -> docker.io/acme/demo\n" + "Service/a -> docker.io/acme/demo:*-a\n" + "Service/b -> docker.io/acme/demo:*-b\n" + "Service/c -> docker.io/acme/demo:*-c\n" + "Service/p -> docker.io/acme/y:*-p\n" + "Service/q -> docker.io/acme/y:*-q\n" + "Service/solo -> docker.io/acme/solo\n" + "Service/tool -> docker.io/acme/tool\n" + "Service/tooltest -> docker.io/acme/tool:test\n" + "Service/v -> docker.io/acme/x:*-v\n" + "Service/w1 -> docker.io/acme/x\n" + "Service/w2 -> docker.io/acme/x"; + if (!got || strcmp(got, want_users) != 0) { + fprintf(stderr, "image users got:\n%s\nwant:\n%s\n", got ? got : "(null)", want_users); + free(got); + FAIL("references on component Images"); + } + free(got); + /* acme/y's repository Image lost every edge to its components: none left */ + int orphans = gpf_query_int(files, NF, + "SELECT COUNT(*) FROM nodes n WHERE n.project=?1 AND " + "n.label='Image' AND NOT EXISTS (SELECT 1 FROM edges e WHERE " + "e.source_id=n.id OR e.target_id=n.id)"); + if (orphans != 0) { + fprintf(stderr, "Images without an edge: %d\n", orphans); + FAIL("a repository Image left without edges"); + } + PASS(); +} + /* Every CONFIGURES {strategy:env_set} edge as * " -> / ()"; then the joins a scoped * EnvVar must not have: "#PORT=" and @@ -4573,6 +4688,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_compose_services_are_resources); RUN_TEST(probe_shell_docker_builds_build_images); RUN_TEST(probe_container_env_sets_scoped_envvars); + RUN_TEST(probe_image_hub_splits_by_component); RUN_TEST(probe_gitops_objects_deploy_their_units); RUN_TEST(probe_shell_kubectl_deploys_what_it_applies); RUN_TEST(probe_shell_helm_deploys_its_chart); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 85eaa8762..26ff25917 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3746,6 +3746,73 @@ TEST(incremental_env_edit_replaces_scoped_envvars) { PASS(); } +/* A repository split by component (pass_docker.c P4.2b). `rename`: component + * b becomes bee -- the split changes, and the reference in compose.run.yaml, + * a file this run does not repair, must leave the old Image `*-b`: a closure + * repair cannot re-point it, so the run rebuilds. Else an edit keeping the + * split: a closure repair. Returns "repo=N b=M bimg=K" (runner's references + * to the repository Image / to `*-b`, `*-b` Images), or NULL. */ +static char *incremental_image_split(bool rename, cbm_incremental_route_t *route) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_imagesplit_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + return NULL; + } + write_temp_file(tmp, "svc/a/Dockerfile", "FROM alpine:3.20\n"); + write_temp_file(tmp, "svc/b/Dockerfile", "FROM alpine:3.20\n"); + write_temp_file(tmp, "compose.yaml", + "services:\n a:\n image: acme/demo:1-a\n build: ./svc/a\n" + " b:\n image: acme/demo:1-b\n build: ./svc/b\n"); + write_temp_file(tmp, "compose.run.yaml", "services:\n runner:\n image: acme/demo:2-b\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/imagesplit.db", tmp); + static const char to_repo[] = + "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.type='USAGE' AND s.name='Service/runner' AND " + "t.name='docker.io/acme/demo'"; + static const char to_b[] = + "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.type='USAGE' AND s.name='Service/runner' AND " + "t.name='docker.io/acme/demo:*-b'"; + static const char b_img[] = + "SELECT COUNT(*) FROM nodes WHERE label='Image' AND name='docker.io/acme/demo:*-b'"; + if (k8s_inc_index(tmp, db_path) != 0 || k8s_inc_count(db_path, to_b) != 1) { + th_rmtree(tmp); + return NULL; + } + write_temp_file(tmp, "compose.yaml", + rename ? "services:\n a:\n image: acme/demo:1-a\n build: ./svc/a\n" + " b:\n image: acme/demo:1-bee\n build: ./svc/b\n" + : "services:\n a:\n image: acme/demo:1-a\n build: ./svc/a\n" + " environment:\n - MODE\n" + " b:\n image: acme/demo:1-b\n build: ./svc/b\n"); + char *out = NULL; + if (k8s_inc_index(tmp, db_path) == 0) { + char buf[64]; + snprintf(buf, sizeof(buf), "repo=%d b=%d bimg=%d", k8s_inc_count(db_path, to_repo), + k8s_inc_count(db_path, to_b), k8s_inc_count(db_path, b_img)); + out = strdup(buf); + } + *route = cbm_pipeline_incremental_test_last_route(); + th_rmtree(tmp); + return out; +} + +TEST(incremental_image_split_change_rebuilds) { + cbm_incremental_route_t route = CBM_INCREMENTAL_ROUTE_NONE; + char *got = incremental_image_split(true, &route); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_NOT_NULL(got); + ASSERT_STR_EQ(got, "repo=1 b=0 bimg=0"); + free(got); + got = incremental_image_split(false, &route); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_NOT_NULL(got); + ASSERT_STR_EQ(got, "repo=0 b=1 bimg=1"); + free(got); + PASS(); +} + /* A compose project's .env feeds its compose file (variables in image, * build): changing it is no closure repair -- the unchanged compose file's * services would keep the old image. It rebuilds; no stale image remains. */ @@ -19513,6 +19580,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_dockerfile_edit_keeps_build_edges); RUN_TEST(incremental_compose_edit_replaces_build_edges); RUN_TEST(incremental_env_edit_replaces_scoped_envvars); + RUN_TEST(incremental_image_split_change_rebuilds); RUN_TEST(incremental_compose_dotenv_change_rebuilds); RUN_TEST(incremental_dockerfile_edit_keeps_docker_build_edges); RUN_TEST(incremental_second_build_declarer_keeps_the_edge);