mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
feat(infra): an Ansible playbook or task file IMPORTS what it includes
Ansible's composition produced no structure: kubespray's 40 playbooks,
roles and task files, tied together by 217 import_playbook /
include_tasks / import_tasks / include_role / import_role / roles:
directives, were unconnected YAML files.
A playbook or task file -- a YAML file whose top level is a sequence;
manifests, values, CI and compose files are mappings -- now IMPORTS
{local_name: <directive>, via: ansible, line} what each directive names:
import_playbook a playbook from its directory; include_tasks /
import_tasks / include a task file from its directory, else from its
role's tasks/; roles: items and include_role / import_role a role's
Folder, roles/<name> at or above its directory (a role by "./" / "../"
path from the directory). The ansible.builtin. prefix is understood; a
templated target links nothing.
Measured on 17 infrastructure repositories: 211 of 211 composition pairs
of kubespray against an independent reading, nothing else in any
corpus; deterministic over three runs; the incremental re-index equals
a full index in 4 scenarios (an included file edited or deleted, an
include pointed elsewhere, a playbook edited).
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
1 parent
fb73491ee8
commit
ce0ed0e3b9
9 files changed
+513
-4
No files matched your search
@@ -540,6 +540,7 @@ PIPELINE_SRCS = \
|
||||
src/pipeline/pass_operator.c \
|
||||
src/pipeline/pass_shell.c \
|
||||
src/pipeline/pass_terraform.c \
|
||||
src/pipeline/pass_ansible.c \
|
||||
src/pipeline/pass_docker.c \
|
||||
src/pipeline/pass_envjoin.c \
|
||||
src/pipeline/pass_ci.c \
|
||||
|
||||
@@ -259,6 +259,7 @@ The install script placed beside the binary is **reported, not deleted** — uni
|
||||
- `USAGE` with `via: terraform` — a Terraform / OpenTofu reference (`var.X`, `local.X`, `module.X`, `data.T.N`, `T.N`) from the top-level block holding it to the block of its module — the `.tf` / `.tofu` files of its directory — that declares it (a `locals` entry: its `locals` block); an entry of the block's `depends_on` list is a `DEPENDS_ON` instead. The edge keeps `ref` (`var`, `local`, `module`, `data`, `resource`) and `line`; a name only mentioned in string text is no reference. A module call with a local source (`module "net" { source = "./modules/net" }`) `IMPORTS` (`local_name`: the module's name) the `Folder` of that directory (the root directory: the `Project`), and `module.net.id` (also `module.net[0].id`, `module.net[*].id`) `USAGE`s (`ref: output`) the `output "id"` there, beside `module.net`
|
||||
- `OVERRIDE` with `via: tfvars` — a `.tfvars` assignment to a variable the module of its own directory declares, from the file's `Variable` to that `variable` block (the edge keeps `line`); a `.tfvars` file elsewhere is applied by a `-var-file` the repository does not state and links nothing
|
||||
- Terragrunt (`via: terragrunt`) — a unit (`terragrunt.hcl`), evaluated in its own context: an `include` `IMPORTS` (`local_name`) the file its `path` names; a `read_terragrunt_config` `IMPORTS` (`read`: the file holding the call) the file it names, from the block holding the call or, for a call in an included file, from the unit's `include` block that brings it in; a `dependency` `DEPENDS_ON` the other unit; a local `terraform` `source` `IMPORTS` its `Folder`. Paths may use `find_in_parent_folders`, `dirname` and `get_terragrunt_dir`; anything else is unknown and links nothing
|
||||
- Ansible (`via: ansible`) — a playbook or task file (a YAML file whose top level is a sequence) `IMPORTS` (`local_name`: the directive) what its directives name: `import_playbook` a playbook, `include_tasks` / `import_tasks` / `include` a task file (from its directory, else its role's `tasks/`), `roles:` items and `include_role` / `import_role` a role's `Folder` (`roles/<name>` at or above its directory). A templated target links nothing
|
||||
- `EMITS`, `LISTENS_ON` (channels)
|
||||
- `DATA_FLOWS` with arg-to-param mapping + field access chains
|
||||
- `SIMILAR_TO` (MinHash + LSH near-clone detection, Jaccard scored)
|
||||
|
||||
@@ -82,7 +82,7 @@ REFS=(
|
||||
|
||||
# Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too).
|
||||
INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c
|
||||
src/pipeline/pass_shell.c src/pipeline/pass_terraform.c
|
||||
src/pipeline/pass_shell.c src/pipeline/pass_terraform.c src/pipeline/pass_ansible.c
|
||||
src/pipeline/pass_ci.c src/pipeline/pass_docker.c src/pipeline/pass_envjoin.c)
|
||||
|
||||
ONLY_LANG=""; REFRESH=0; LIST_ONLY=0
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
/*
|
||||
* pass_ansible.c — the Ansible layer.
|
||||
*
|
||||
* PHASE 7 P7.1: composition. A playbook or a task file -- a YAML file whose
|
||||
* top level is a sequence (manifests, values, CI and compose files are
|
||||
* mappings) -- IMPORTS {local_name: <directive>, via: ansible, line} what its
|
||||
* directives name:
|
||||
* import_playbook: P the playbook P, from the file's
|
||||
* directory
|
||||
* include_tasks / import_tasks /
|
||||
* include: T (or {file: T}) the task file T, from the file's
|
||||
* directory, else from the tasks/
|
||||
* directory of the role holding it
|
||||
* include_role / import_role: {name: R},
|
||||
* roles: [R | {role: R} | {name: R}] the role R: roles/R in the file's
|
||||
* directory or above (a name may hold
|
||||
* "/", roles/kubernetes/node), or a
|
||||
* path ("./", "../") from the
|
||||
* file's directory
|
||||
* (the ansible.builtin. prefix allowed). The edge goes from the file to the
|
||||
* target file's Module, or to the role's Folder; a target the graph does not
|
||||
* hold, or one holding a Jinja expression ({{ }}), makes none.
|
||||
*/
|
||||
#include "foundation/constants.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/hash_table.h"
|
||||
#include "foundation/limits.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/mem_core.h"
|
||||
#include "graph_buffer/graph_buffer.h"
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include "cbm.h"
|
||||
#include "helpers.h"
|
||||
#include "lang_specs.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
cbm_pipeline_ctx_t *ctx;
|
||||
CBMHashTable *modules; /* file path -> its Module node */
|
||||
CBMHashTable *folders; /* directory -> its Folder node */
|
||||
const char *src; /* the file's text */
|
||||
const char *file; /* its repository path */
|
||||
char dir[CBM_SZ_1K]; /* its directory */
|
||||
char role[CBM_SZ_1K]; /* the role holding it (roles/<name>), or "" */
|
||||
int64_t from; /* its File node */
|
||||
int edges;
|
||||
} an_t;
|
||||
|
||||
static bool an_type(TSNode n, const char *t) {
|
||||
return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0;
|
||||
}
|
||||
|
||||
/* A scalar's text (quotes stripped) into out; false for anything else. */
|
||||
static bool an_scalar(const an_t *a, TSNode n, char *out, size_t sz) {
|
||||
while ((an_type(n, "flow_node") || an_type(n, "block_node") || an_type(n, "plain_scalar")) &&
|
||||
ts_node_named_child_count(n) == SKIP_ONE) {
|
||||
n = ts_node_named_child(n, 0);
|
||||
}
|
||||
if (ts_node_is_null(n)) { /* a pair without its key or value */
|
||||
return false;
|
||||
}
|
||||
uint32_t b = ts_node_start_byte(n);
|
||||
uint32_t e = ts_node_end_byte(n);
|
||||
if (an_type(n, "double_quote_scalar") || an_type(n, "single_quote_scalar")) {
|
||||
b++;
|
||||
e = e > b ? e - SKIP_ONE : b;
|
||||
} else if (!an_type(n, "string_scalar") && !an_type(n, "plain_scalar")) {
|
||||
return false;
|
||||
}
|
||||
if (e - b + SKIP_ONE > sz) {
|
||||
return false;
|
||||
}
|
||||
memcpy(out, a->src + b, e - b);
|
||||
out[e - b] = '\0';
|
||||
return out[0] != '\0';
|
||||
}
|
||||
|
||||
/* The key of a mapping pair, its text, the ansible.builtin. prefix dropped. */
|
||||
static bool an_key(const an_t *a, TSNode pair, char *out, size_t sz) {
|
||||
static const char BUILTIN[] = "ansible.builtin.";
|
||||
if (!an_scalar(a, ts_node_child_by_field_name(pair, "key", sizeof("key") - SKIP_ONE), out,
|
||||
sz)) {
|
||||
return false;
|
||||
}
|
||||
size_t bl = sizeof(BUILTIN) - SKIP_ONE;
|
||||
if (strncmp(out, BUILTIN, bl) == 0) {
|
||||
memmove(out, out + bl, strlen(out + bl) + SKIP_ONE);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The value of `key` in a block mapping (under its block_node), or a null node. */
|
||||
static TSNode an_get(const an_t *a, TSNode value, const char *key) {
|
||||
while (an_type(value, "block_node") && ts_node_named_child_count(value) == SKIP_ONE) {
|
||||
value = ts_node_named_child(value, 0);
|
||||
}
|
||||
for (uint32_t i = 0; an_type(value, "block_mapping") && i < ts_node_named_child_count(value);
|
||||
i++) {
|
||||
TSNode pair = ts_node_named_child(value, i);
|
||||
char k[CBM_SZ_64];
|
||||
if (an_type(pair, "block_mapping_pair") && an_key(a, pair, k, sizeof(k)) &&
|
||||
strcmp(k, key) == 0) {
|
||||
return ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE);
|
||||
}
|
||||
}
|
||||
return (TSNode){0};
|
||||
}
|
||||
|
||||
static void an_edge(an_t *a, const cbm_gbuf_node_t *tgt, const char *kind, TSNode pair) {
|
||||
if (!tgt || a->from <= 0 || tgt->id == a->from) {
|
||||
return;
|
||||
}
|
||||
char props[CBM_SZ_128];
|
||||
snprintf(props, sizeof(props), "{\"local_name\":\"%s\",\"via\":\"ansible\",\"line\":%d}", kind,
|
||||
(int)ts_node_start_point(pair).row + SKIP_ONE);
|
||||
cbm_gbuf_insert_edge(a->ctx->gbuf, a->from, tgt->id, "IMPORTS", props);
|
||||
a->edges++;
|
||||
}
|
||||
|
||||
/* A file `v` from `base` (repository-relative), its Module; NULL when none. */
|
||||
static const cbm_gbuf_node_t *an_file(const an_t *a, const char *base, const char *v) {
|
||||
char path[CBM_SZ_1K];
|
||||
if (!cbm_repo_rel_join(base, v, path, sizeof(path))) {
|
||||
return NULL;
|
||||
}
|
||||
return cbm_ht_get(a->modules, path);
|
||||
}
|
||||
|
||||
/* Role `name`'s Folder: a path ("./", "../") from the file's directory, else
|
||||
* roles/<name> in that directory or above. */
|
||||
static const cbm_gbuf_node_t *an_role(const an_t *a, const char *name) {
|
||||
char path[CBM_SZ_1K];
|
||||
if (name[0] == '.' || name[0] == '/') {
|
||||
return cbm_repo_rel_join(a->dir, name, path, sizeof(path)) ? cbm_ht_get(a->folders, path)
|
||||
: NULL;
|
||||
}
|
||||
char d[CBM_SZ_1K];
|
||||
snprintf(d, sizeof(d), "%s", a->dir);
|
||||
for (;;) {
|
||||
snprintf(path, sizeof(path), "%s%sroles/%s", d, d[0] ? "/" : "", name);
|
||||
const cbm_gbuf_node_t *f = cbm_ht_get(a->folders, path);
|
||||
if (f || !d[0]) {
|
||||
return f;
|
||||
}
|
||||
char *s = strrchr(d, '/');
|
||||
if (s) {
|
||||
*s = '\0';
|
||||
} else {
|
||||
d[0] = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* One mapping pair of a task or a play. */
|
||||
static void an_pair(an_t *a, TSNode pair) {
|
||||
char key[CBM_SZ_64];
|
||||
char v[CBM_SZ_1K];
|
||||
if (!an_key(a, pair, key, sizeof(key))) {
|
||||
return;
|
||||
}
|
||||
TSNode value = ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE);
|
||||
if (strcmp(key, "import_playbook") == 0) {
|
||||
if (an_scalar(a, value, v, sizeof(v)) && !strstr(v, "{{")) {
|
||||
an_edge(a, an_file(a, a->dir, v), key, pair);
|
||||
}
|
||||
} else if (strcmp(key, "include_tasks") == 0 || strcmp(key, "import_tasks") == 0 ||
|
||||
strcmp(key, "include") == 0) {
|
||||
if ((an_scalar(a, value, v, sizeof(v)) ||
|
||||
an_scalar(a, an_get(a, value, "file"), v, sizeof(v))) &&
|
||||
!strstr(v, "{{")) {
|
||||
char tasks[CBM_SZ_1K];
|
||||
snprintf(tasks, sizeof(tasks), "%s/tasks", a->role);
|
||||
const cbm_gbuf_node_t *t = an_file(a, a->dir, v);
|
||||
an_edge(a, t ? t : a->role[0] ? an_file(a, tasks, v) : NULL, key, pair);
|
||||
}
|
||||
} else if (strcmp(key, "include_role") == 0 || strcmp(key, "import_role") == 0) {
|
||||
if (an_scalar(a, an_get(a, value, "name"), v, sizeof(v)) && !strstr(v, "{{")) {
|
||||
an_edge(a, an_role(a, v), key, pair);
|
||||
}
|
||||
} else if (strcmp(key, "roles") == 0) {
|
||||
while (an_type(value, "block_node") && ts_node_named_child_count(value) == SKIP_ONE) {
|
||||
value = ts_node_named_child(value, 0);
|
||||
}
|
||||
for (uint32_t i = 0;
|
||||
an_type(value, "block_sequence") && i < ts_node_named_child_count(value); i++) {
|
||||
TSNode item = ts_node_named_child(ts_node_named_child(value, i), 0);
|
||||
if ((an_scalar(a, item, v, sizeof(v)) ||
|
||||
an_scalar(a, an_get(a, item, "role"), v, sizeof(v)) ||
|
||||
an_scalar(a, an_get(a, item, "name"), v, sizeof(v))) &&
|
||||
!strstr(v, "{{")) {
|
||||
an_edge(a, an_role(a, v), key, ts_node_named_child(value, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Every mapping that is a sequence item (a play or a task, also one nested in
|
||||
* block: / tasks: / handlers: ...): its pairs. */
|
||||
static void an_walk(an_t *a, TSNode root) {
|
||||
TSTreeCursor cur = ts_tree_cursor_new(root);
|
||||
bool more = true;
|
||||
while (more) {
|
||||
TSNode c = ts_tree_cursor_current_node(&cur);
|
||||
if (an_type(c, "block_mapping")) {
|
||||
TSNode up = ts_node_parent(ts_node_parent(c));
|
||||
for (uint32_t i = 0;
|
||||
an_type(up, "block_sequence_item") && i < ts_node_named_child_count(c); i++) {
|
||||
if (an_type(ts_node_named_child(c, i), "block_mapping_pair")) {
|
||||
an_pair(a, ts_node_named_child(c, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ts_tree_cursor_goto_first_child(&cur)) {
|
||||
continue;
|
||||
}
|
||||
while (!ts_tree_cursor_goto_next_sibling(&cur)) {
|
||||
if (!ts_tree_cursor_goto_parent(&cur)) {
|
||||
more = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
ts_tree_cursor_delete(&cur);
|
||||
}
|
||||
|
||||
/* The first content line of `src` starts a sequence item: Ansible content
|
||||
* (a playbook, a task list), not a mapping (manifest, values, CI, compose). */
|
||||
static bool an_is_sequence(const char *src) {
|
||||
for (const char *p = src; *p;) {
|
||||
const char *nl = strchr(p, '\n');
|
||||
size_t n = nl ? (size_t)(nl - p) : strlen(p);
|
||||
size_t s = 0;
|
||||
while (s < n && (p[s] == ' ' || p[s] == '\t' || p[s] == '\r')) {
|
||||
s++;
|
||||
}
|
||||
bool skip = s == n || p[s] == '#' ||
|
||||
(n - s >= sizeof("---") - SKIP_ONE &&
|
||||
strncmp(p + s, "---", sizeof("---") - SKIP_ONE) == 0);
|
||||
if (!skip) {
|
||||
return s == 0 && n >= PAIR_LEN && p[0] == '-' && p[SKIP_ONE] == ' ';
|
||||
}
|
||||
p = nl ? nl + SKIP_ONE : p + n;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* The role holding `rel`: roles/<name> of the deepest "roles" component with
|
||||
* a name after it, or "". */
|
||||
static void an_role_of(const char *rel, char *out, size_t sz) {
|
||||
out[0] = '\0';
|
||||
for (const char *p = rel; (p = strstr(p, "roles/")) != NULL; p++) {
|
||||
if (p != rel && p[-1] != '/') {
|
||||
continue;
|
||||
}
|
||||
const char *name = p + sizeof("roles/") - SKIP_ONE;
|
||||
const char *end = strchr(name, '/');
|
||||
if (end && strchr(end + SKIP_ONE, '/')) {
|
||||
snprintf(out, sz, "%.*s", (int)(end - rel), rel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static char *an_read(const char *path, size_t *len) {
|
||||
FILE *f = cbm_fopen(path, "rb");
|
||||
if (!f) {
|
||||
return NULL;
|
||||
}
|
||||
(void)fseek(f, 0, SEEK_END);
|
||||
long size = ftell(f);
|
||||
(void)fseek(f, 0, SEEK_SET);
|
||||
if (size <= 0 || size > cbm_max_file_bytes()) {
|
||||
(void)fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
enum { AN_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */
|
||||
char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + AN_TS_PAD);
|
||||
if (!buf) {
|
||||
(void)fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
size_t got = fread(buf, SKIP_ONE, (size_t)size, f);
|
||||
(void)fclose(f);
|
||||
memset(buf + got, 0, AN_TS_PAD);
|
||||
*len = got;
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* path -> node index of one label's nodes. */
|
||||
static CBMHashTable *an_index(cbm_pipeline_ctx_t *ctx, const char *label) {
|
||||
CBMHashTable *ix = cbm_ht_create(0);
|
||||
const cbm_gbuf_node_t **nodes = NULL;
|
||||
int n = 0;
|
||||
if (ix && cbm_gbuf_find_by_label(ctx->gbuf, label, &nodes, &n) == 0) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (nodes[i]->file_path) {
|
||||
cbm_ht_set(ix, nodes[i]->file_path, (void *)nodes[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ix;
|
||||
}
|
||||
|
||||
static bool an_is_yaml(const char *rel) {
|
||||
const char *dot = strrchr(rel, '.');
|
||||
return dot && (strcmp(dot, ".yml") == 0 || strcmp(dot, ".yaml") == 0);
|
||||
}
|
||||
|
||||
void cbm_pipeline_pass_ansible(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
int file_count) {
|
||||
an_t a = {.ctx = ctx};
|
||||
TSParser *parser = NULL;
|
||||
CBMHashTable *file_nodes = NULL;
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (!an_is_yaml(files[i].rel_path)) {
|
||||
continue;
|
||||
}
|
||||
size_t len = 0;
|
||||
char *src = an_read(files[i].path, &len);
|
||||
if (!src || !an_is_sequence(src)) {
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, src);
|
||||
continue;
|
||||
}
|
||||
if (!parser) {
|
||||
parser = ts_parser_new();
|
||||
if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) {
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, src);
|
||||
break;
|
||||
}
|
||||
a.modules = an_index(ctx, "Module");
|
||||
a.folders = an_index(ctx, "Folder");
|
||||
file_nodes = an_index(ctx, "File");
|
||||
}
|
||||
const cbm_gbuf_node_t *fnode =
|
||||
file_nodes ? cbm_ht_get(file_nodes, files[i].rel_path) : NULL;
|
||||
TSTree *tree = fnode ? ts_parser_parse_string(parser, NULL, src, (uint32_t)len) : NULL;
|
||||
if (tree) {
|
||||
const char *rel = files[i].rel_path;
|
||||
const char *slash = strrchr(rel, '/');
|
||||
a.src = src;
|
||||
a.file = rel;
|
||||
a.from = fnode->id;
|
||||
snprintf(a.dir, sizeof(a.dir), "%.*s", slash ? (int)(slash - rel) : 0, rel);
|
||||
an_role_of(rel, a.role, sizeof(a.role));
|
||||
an_walk(&a, ts_tree_root_node(tree));
|
||||
ts_tree_delete(tree);
|
||||
}
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, src);
|
||||
}
|
||||
if (parser) {
|
||||
ts_parser_delete(parser);
|
||||
}
|
||||
if (a.modules) {
|
||||
cbm_ht_free(a.modules);
|
||||
}
|
||||
if (a.folders) {
|
||||
cbm_ht_free(a.folders);
|
||||
}
|
||||
if (file_nodes) {
|
||||
cbm_ht_free(file_nodes);
|
||||
}
|
||||
if (a.edges > 0) {
|
||||
char buf[CBM_SZ_32];
|
||||
snprintf(buf, sizeof(buf), "%d", a.edges);
|
||||
cbm_log_info("pass.ansible.edges", "imports", buf);
|
||||
}
|
||||
}
|
||||
@@ -4154,6 +4154,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
/* Shell and Make deploy commands, against the nodes linked above. */
|
||||
cbm_pipeline_pass_shell(ctx, files, file_count);
|
||||
cbm_pipeline_pass_terraform(ctx, files, file_count);
|
||||
cbm_pipeline_pass_ansible(ctx, files, file_count);
|
||||
cbm_pipeline_pass_docker(ctx, files, file_count);
|
||||
/* The variables a container sets, to the readers in its own image. */
|
||||
(void)cbm_pipeline_env_join(ctx);
|
||||
|
||||
@@ -182,6 +182,10 @@ static bool tf_repo_path(const char *mdir, const char *v, char *out, size_t sz)
|
||||
return true;
|
||||
}
|
||||
|
||||
bool cbm_repo_rel_join(const char *dir, const char *v, char *out, size_t sz) {
|
||||
return tf_repo_path(dir, v, out, sz);
|
||||
}
|
||||
|
||||
/* Chart.yaml path -> its (local) Chart node. */
|
||||
static CBMHashTable *tf_chart_index(cbm_pipeline_ctx_t *ctx) {
|
||||
CBMHashTable *ix = cbm_ht_create(0);
|
||||
|
||||
@@ -1298,11 +1298,23 @@ bool cbm_build_path(const char *rel);
|
||||
void cbm_build_sites(CBMArena *a, const char *rel, const char *src, size_t len, const char *dotenv,
|
||||
cbm_build_site_fn fn, void *ud);
|
||||
|
||||
/* Terraform / OpenTofu modules (pass_terraform.c): a helm_release DEPLOYS its
|
||||
* chart. Run at the end of the K8s pass, over the same files. */
|
||||
/* Terraform / OpenTofu / Terragrunt (pass_terraform.c): references, module
|
||||
* calls, tfvars, Terragrunt units; a helm_release DEPLOYS its chart. Run at
|
||||
* the end of the K8s pass, over the same files. */
|
||||
void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
int file_count);
|
||||
|
||||
/* Ansible (pass_ansible.c): a playbook or task file IMPORTS the playbooks,
|
||||
* task files and roles its directives name. Run at the end of the K8s pass,
|
||||
* over the same files. */
|
||||
void cbm_pipeline_pass_ansible(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
int file_count);
|
||||
|
||||
/* `v` (a path; "./" and "../" allowed) from repository directory `dir`, as a
|
||||
* repository path into out; false past the root or when it does not fit
|
||||
* (pass_terraform.c). */
|
||||
bool cbm_repo_rel_join(const char *dir, const char *v, char *out, size_t sz);
|
||||
|
||||
/* The Image node of image reference `ref` (its normalised repository:
|
||||
* docker.io for no registry host, library/ for an official image; tag and
|
||||
* digest dropped), made when new; 0 for scratch or no image. */
|
||||
@@ -1508,7 +1520,9 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr
|
||||
* {via:tfvars} the variable block of its own directory; a Terragrunt
|
||||
* unit's include / read_terragrunt_config IMPORTS {via:terragrunt} the
|
||||
* file it names, a dependency DEPENDS_ON the other unit
|
||||
* (pass_terraform.c, extract_unified.c, extract_defs.c). An index built
|
||||
* (pass_terraform.c, extract_unified.c, extract_defs.c); an Ansible
|
||||
* playbook / task file IMPORTS {via:ansible} the playbooks, task files
|
||||
* and roles its directives name (pass_ansible.c). An index built
|
||||
* before holds none of them, so it rebuilds once on upgrade. */
|
||||
enum { CBM_SEMANTIC_INDEX_VERSION = 9 };
|
||||
|
||||
|
||||
@@ -3098,6 +3098,79 @@ TEST(probe_terragrunt_units_compose_their_files) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Every Ansible IMPORTS edge as "<file> <type> <label>:<target> <local_name>:<line>". */
|
||||
static char *gpf_ansible(const GpfFile *files, int nfiles) {
|
||||
GpfProj lp;
|
||||
cbm_store_t *store = gpf_index_files(&lp, files, nfiles);
|
||||
char *out = NULL;
|
||||
if (store) {
|
||||
sqlite3_stmt *st = NULL;
|
||||
const char *sql =
|
||||
"SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || "
|
||||
"e.type || ' ' || t.label || ':' || t.file_path || ' ' || "
|
||||
"json_extract(e.properties,'$.local_name') || ':' || "
|
||||
"json_extract(e.properties,'$.line') AS x FROM edges e JOIN nodes s ON "
|
||||
"s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND "
|
||||
"json_extract(e.properties,'$.via')='ansible' ORDER BY x)), '')";
|
||||
if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) {
|
||||
sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC);
|
||||
if (sqlite3_step(st) == SQLITE_ROW) {
|
||||
out = strdup((const char *)sqlite3_column_text(st, 0));
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(st);
|
||||
}
|
||||
gpf_cleanup(&lp, store);
|
||||
return out;
|
||||
}
|
||||
|
||||
/* A playbook or task file (a YAML file whose top level is a sequence)
|
||||
* IMPORTS {local_name: <directive>} what its directives name: import_playbook
|
||||
* a playbook from its directory; include_tasks / import_tasks a task file
|
||||
* from its directory, else from its role's tasks/ (also inside a block:);
|
||||
* roles: items and include_role / import_role a role's Folder, roles/<name>
|
||||
* at or above its directory. A templated target, an empty directive, and a
|
||||
* mapping-topped file (values, CI) with the same keys, make none. */
|
||||
TEST(probe_ansible_files_import_what_they_name) {
|
||||
static const GpfFile files[] = {
|
||||
{"site.yml", "- name: Install\n import_playbook: playbooks/cluster.yml\n"},
|
||||
{"playbooks/cluster.yml",
|
||||
"- hosts: all\n roles:\n - common\n - role: web\n when: false\n"
|
||||
" - \"{{ dyn }}\"\n tasks:\n - name: a\n include_tasks: tasks/extra.yml\n"
|
||||
" - name: b\n ansible.builtin.import_role:\n name: db\n"},
|
||||
{"playbooks/tasks/extra.yml", "- name: x\n debug:\n msg: hi\n"},
|
||||
{"roles/common/tasks/main.yml",
|
||||
"- name: setup\n include_tasks: setup.yml\n- name: nested\n block:\n"
|
||||
" - name: inner\n import_tasks: other.yml\n"},
|
||||
{"roles/common/tasks/setup.yml", "- name: s\n debug:\n msg: s\n"},
|
||||
{"roles/common/tasks/other.yml", "- name: o\n debug:\n msg: o\n"},
|
||||
{"roles/common/handlers/main.yml", "- name: h\n include_tasks: setup.yml\n"},
|
||||
{"roles/web/tasks/main.yml",
|
||||
"- name: w\n debug:\n msg: w\n- name: empty\n include_tasks:\n"},
|
||||
{"roles/db/tasks/main.yml", "- name: d\n debug:\n msg: d\n"},
|
||||
{"values.yaml", "include: playbooks/tasks/extra.yml\n"
|
||||
"users:\n - name: ops\n roles:\n - common\n"}};
|
||||
int n = (int)(sizeof(files) / sizeof(files[0]));
|
||||
char *got = gpf_ansible(files, n);
|
||||
static const char want[] =
|
||||
"playbooks/cluster.yml IMPORTS Folder:roles/common roles:3\n"
|
||||
"playbooks/cluster.yml IMPORTS Folder:roles/db import_role:11\n"
|
||||
"playbooks/cluster.yml IMPORTS Folder:roles/web roles:4\n"
|
||||
"playbooks/cluster.yml IMPORTS Module:playbooks/tasks/extra.yml include_tasks:9\n"
|
||||
"roles/common/handlers/main.yml IMPORTS Module:roles/common/tasks/setup.yml "
|
||||
"include_tasks:2\n"
|
||||
"roles/common/tasks/main.yml IMPORTS Module:roles/common/tasks/other.yml import_tasks:6\n"
|
||||
"roles/common/tasks/main.yml IMPORTS Module:roles/common/tasks/setup.yml include_tasks:2\n"
|
||||
"site.yml IMPORTS Module:playbooks/cluster.yml import_playbook:2";
|
||||
if (!got || strcmp(got, want) != 0) {
|
||||
fprintf(stderr, "ansible got:\n%s\nwant:\n%s\n", got ? got : "(null)", want);
|
||||
free(got);
|
||||
FAIL("ansible");
|
||||
}
|
||||
free(got);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Every script HTTP_CALLS edge as "<source file> <route QN>", then a line
|
||||
* "edgeless routes <n>": the Route nodes no edge touches. */
|
||||
static char *gpf_script_http(const GpfFile *files, int nfiles) {
|
||||
@@ -5540,6 +5613,7 @@ SUITE(grammar_probe_f) {
|
||||
RUN_TEST(probe_terraform_module_calls_import_their_source);
|
||||
RUN_TEST(probe_terraform_tfvars_override_their_variables);
|
||||
RUN_TEST(probe_terragrunt_units_compose_their_files);
|
||||
RUN_TEST(probe_ansible_files_import_what_they_name);
|
||||
RUN_TEST(probe_image_hub_splits_by_component);
|
||||
RUN_TEST(probe_gitops_objects_deploy_their_units);
|
||||
RUN_TEST(probe_shell_kubectl_deploys_what_it_applies);
|
||||
|
||||
@@ -1492,6 +1492,47 @@ TEST(pipeline_incremental_preserves_cross_file_calls) {
|
||||
|
||||
static void write_temp_file(const char *dir, const char *name, const char *content);
|
||||
|
||||
/* An Ansible task file included by an UNCHANGED file is edited, its body
|
||||
* only. The includer is not re-parsed, so its IMPORTS edge must survive the
|
||||
* edited file's re-index: an edge into a node the repair re-creates would be
|
||||
* lost (the Terraform layer needed closure_terraform_decline for that). */
|
||||
TEST(pipeline_incremental_keeps_ansible_includes_of_an_edited_file) {
|
||||
if (setup_test_repo() != 0) {
|
||||
FAIL("failed to create temp dir");
|
||||
}
|
||||
write_temp_file(g_tmpdir, "roles/app/tasks/main.yml",
|
||||
"- name: setup\n include_tasks: setup.yml\n");
|
||||
write_temp_file(g_tmpdir, "roles/app/tasks/setup.yml", "- name: s\n debug:\n msg: one\n");
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/test_incr_ansible.db", g_tmpdir);
|
||||
|
||||
cbm_pipeline_t *p1 = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p1);
|
||||
ASSERT_EQ(cbm_pipeline_run(p1), 0);
|
||||
cbm_store_t *s1 = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(s1);
|
||||
ASSERT_TRUE(cross_file_edge_exists(s1, cbm_pipeline_project_name(p1), "main.yml",
|
||||
"roles/app/tasks/setup.yml", "IMPORTS"));
|
||||
cbm_store_close(s1);
|
||||
cbm_pipeline_free(p1);
|
||||
|
||||
write_temp_file(g_tmpdir, "roles/app/tasks/setup.yml",
|
||||
"- name: s\n debug:\n msg: two words\n");
|
||||
|
||||
cbm_pipeline_t *p2 = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p2);
|
||||
ASSERT_EQ(cbm_pipeline_run(p2), 0);
|
||||
cbm_store_t *s2 = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(s2);
|
||||
ASSERT_TRUE(cross_file_edge_exists(s2, cbm_pipeline_project_name(p2), "main.yml",
|
||||
"roles/app/tasks/setup.yml", "IMPORTS"));
|
||||
cbm_store_close(s2);
|
||||
cbm_pipeline_free(p2);
|
||||
|
||||
teardown_test_repo();
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Regression: a Terraform module's output read lives in one file
|
||||
* (infra/outputs.tf: module.net.vpc_id) and the module's source in another
|
||||
* (infra/main.tf). Pointing the source elsewhere edits main.tf alone; the
|
||||
@@ -23193,6 +23234,7 @@ SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_hcl_block_reference_resolves_to_its_block);
|
||||
RUN_TEST(pipeline_incremental_preserves_cross_file_calls);
|
||||
RUN_TEST(pipeline_incremental_drops_terraform_reads_of_a_moved_module);
|
||||
RUN_TEST(pipeline_incremental_keeps_ansible_includes_of_an_edited_file);
|
||||
RUN_TEST(pipeline_field_named_like_method_keeps_its_reads_and_writes);
|
||||
RUN_TEST(pipeline_objectscript_export_preserves_calls_sequential_parallel);
|
||||
RUN_TEST(pipeline_objectscript_export_incremental_matches_full_relationships);
|
||||
|
||||
Reference in new issue
Block a user