diff --git a/.codebase-memory.json b/.codebase-memory.json index 855970754..448c14f6f 100644 --- a/.codebase-memory.json +++ b/.codebase-memory.json @@ -44,6 +44,16 @@ "action": "ignore" } ], - "unmapped_files": "run_all" + "unmapped_files": "run_all", + "coverage": { + "compatibility_paths": [ + "tests/test_framework.h", + "tests/test_main.c", + "tests/test_helpers.h", + "Makefile.cbm", + "scripts/test-impact/coverage-map.py", + "scripts/test-impact/coverage-merge.py" + ] + } } } diff --git a/Makefile.cbm b/Makefile.cbm index 3c4e3d2a5..4cf16f90e 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -422,14 +422,14 @@ PREPROCESSOR_SRC = $(CBM_DIR)/preprocessor.cpp SQLITE_WRITER_SRC = $(CBM_DIR)/sqlite_writer.c # Store module (new) -STORE_SRCS = src/store/store.c src/store/store_impact.c src/store/store_graph_digest.c +STORE_SRCS = src/store/store.c src/store/store_impact.c src/store/store_graph_digest.c src/store/store_content_digest.c # Cypher module (new) CYPHER_SRCS = src/cypher/cypher.c # MCP server module (new) MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c src/mcp/compact_out.c \ - src/mcp/test_impact.c src/mcp/test_impact_diff.c src/mcp/test_impact_classify.c src/mcp/test_impact_source.c src/mcp/test_impact_seed.c src/mcp/test_impact_engine.c src/mcp/test_impact_engine_snapshot.c src/mcp/test_impact_engine_reach.c src/mcp/test_impact_changes.c src/mcp/test_impact_origins.c src/mcp/test_impact_coverage.c src/mcp/test_impact_config.c src/mcp/test_impact_select.c src/mcp/test_impact_policy.c src/mcp/test_impact_git.c src/mcp/test_impact_inventory.c src/mcp/test_impact_inventory_input.c src/mcp/test_impact_tree.c src/mcp/test_impact_tree_audit.c src/mcp/test_impact_tree_read.c src/mcp/test_impact_tree_batch.c src/mcp/test_impact_tree_digest.c src/mcp/test_impact_tree_names.c src/mcp/test_impact_tree_plan.c src/mcp/test_impact_tree_posix.c src/mcp/test_impact_tree_posix_acl.c src/mcp/test_impact_tree_posix_io.c src/mcp/test_impact_tree_util.c src/mcp/test_impact_tree_windows.c src/mcp/test_impact_tree_windows_acl.c src/mcp/test_impact_tree_windows_io.c \ + src/mcp/test_impact.c src/mcp/test_impact_diff.c src/mcp/test_impact_classify.c src/mcp/test_impact_source.c src/mcp/test_impact_seed.c src/mcp/test_impact_engine.c src/mcp/test_impact_engine_snapshot.c src/mcp/test_impact_engine_reach.c src/mcp/test_impact_engine_artifact.c src/mcp/test_impact_cli.c src/mcp/test_impact_artifact.c src/mcp/test_impact_changes.c src/mcp/test_impact_origins.c src/mcp/test_impact_coverage.c src/mcp/test_impact_config.c src/mcp/test_impact_select.c src/mcp/test_impact_policy.c src/mcp/test_impact_git.c src/mcp/test_impact_inventory.c src/mcp/test_impact_inventory_input.c src/mcp/test_impact_tree.c src/mcp/test_impact_tree_audit.c src/mcp/test_impact_tree_read.c src/mcp/test_impact_tree_batch.c src/mcp/test_impact_tree_digest.c src/mcp/test_impact_tree_names.c src/mcp/test_impact_tree_plan.c src/mcp/test_impact_tree_posix.c src/mcp/test_impact_tree_posix_acl.c src/mcp/test_impact_tree_posix_io.c src/mcp/test_impact_tree_util.c src/mcp/test_impact_tree_windows.c src/mcp/test_impact_tree_windows_acl.c src/mcp/test_impact_tree_windows_io.c \ src/mcp/test_impact_result.c src/mcp/test_impact_result_json.c src/mcp/test_impact_result_project.c src/mcp/test_impact_result_receipt.c src/mcp/test_impact_result_rows.c src/mcp/test_impact_result_util.c src/mcp/test_impact_result_validate.c src/mcp/test_impact_result_wire.c # Shared MCP daemon coordination and local transport @@ -709,6 +709,7 @@ TEST_MCP_SRCS = \ tests/test_test_impact_source.c \ tests/test_test_impact_seed.c \ tests/test_test_impact_engine.c \ + tests/test_test_impact_artifact.c \ tests/test_test_impact_inventory_errors.c \ tests/test_test_impact_inventory_support.c \ tests/test_test_impact_runner_filter.c \ diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index c5cbe0727..cf3dabf95 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -59,7 +59,6 @@ src/pipeline/lsp_resolve.h 4 src/pipeline/lsp_surface.c 5 src/pipeline/pass_calls.c 22 src/pipeline/pass_compile_commands.c 17 -src/pipeline/pass_complexity.c 15 src/pipeline/pass_configlink.c 3 src/pipeline/pass_cross_repo.c 8 src/pipeline/pass_definitions.c 11 diff --git a/scripts/test-impact/coverage-merge.py b/scripts/test-impact/coverage-merge.py new file mode 100644 index 000000000..1c4cfd7f9 --- /dev/null +++ b/scripts/test-impact/coverage-merge.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""coverage-merge.py — the incremental coverage map of a main-branch push. + +Merges FRESH (coverage-map.py output for the suites the push re-ran, made with +the new commit's instrumented runner, so its functions.tsv is the new image's +complete function table) into PREVIOUS (the previous published map): + + * a suite FRESH ran keeps FRESH's rows exactly; + * every other suite the new runner still lists is carried from PREVIOUS, + each row's function ids remapped by (source file, name) onto FRESH's + table; a function the new image no longer has drops out of the row, and a + row's status (an incomplete row stays incomplete) is never improved; + * a PREVIOUS suite the new runner no longer lists is dropped. + +The merged meta.json names --commit and recounts every suite, so the C reader +(cbm_coverage_map_metadata_matches) checks it like a full map. When the map was +observed is not decided here: the producer carries the oldest observation of +the previous bundle forward, so an incremental map ages out unless a full +refresh runs (the receipt's 7-day bound). + +Usage: coverage-merge.py --previous DIR --fresh DIR --suites FILE --commit SHA --out DIR +Exit: 0 merged · 1 inconsistent inputs · 2 usage. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys + +SETUP = "*" + + +def fail(message: str) -> None: + print(f"coverage-merge: {message}", file=sys.stderr) + sys.exit(1) + + +def read_functions(path: str) -> tuple[dict[int, tuple[str, str]], list[str]]: + by_id: dict[int, tuple[str, str]] = {} + lines = [] + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.rstrip("\n") + if not line: + continue + fields = line.split("\t") + if len(fields) != 3 or not fields[0].isdigit(): + fail(f"malformed function row in {path}: {line!r}") + by_id[int(fields[0])] = (fields[1], fields[2]) + lines.append(line) + return by_id, lines + + +def read_tests(path: str) -> list[tuple[str, str, str, str, list[int]]]: + rows = [] + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.rstrip("\n") + if not line: + continue + fields = line.split("\t") + if len(fields) != 4 or ":" not in fields[0]: + fail(f"malformed test row in {path}: {line!r}") + suite, test = fields[0].split(":", 1) + ids = [int(token) for token in fields[3].split()] if fields[3] else [] + rows.append((suite, test, fields[1], fields[2], ids)) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + for flag in ("previous", "fresh", "suites", "commit", "out"): + parser.add_argument("--" + flag, required=True) + args = parser.parse_args() + + _, fresh_lines = read_functions(os.path.join(args.fresh, "functions.tsv")) + fresh_by_id, _ = read_functions(os.path.join(args.fresh, "functions.tsv")) + previous_by_id, _ = read_functions(os.path.join(args.previous, "functions.tsv")) + fresh_ids = {key: ident for ident, key in fresh_by_id.items()} + with open(os.path.join(args.fresh, "meta.json"), encoding="utf-8") as handle: + fresh_meta = json.load(handle) + with open(os.path.join(args.previous, "meta.json"), encoding="utf-8") as handle: + previous_meta = json.load(handle) + with open(args.suites, encoding="utf-8") as handle: + listed = [line.strip() for line in handle if line.strip()] + + fresh_rows = read_tests(os.path.join(args.fresh, "tests.tsv")) + previous_rows = read_tests(os.path.join(args.previous, "tests.tsv")) + rerun = {row[0] for row in fresh_rows} + if not rerun <= set(listed): + fail("a re-run suite is not listed by the new runner") + + merged = [row for row in fresh_rows] + for suite, test, status, reason, ids in previous_rows: + if suite in rerun or suite not in listed: + continue + mapped = sorted({fresh_ids[previous_by_id[i]] for i in ids + if i in previous_by_id and previous_by_id[i] in fresh_ids}) + merged.append((suite, test, status, reason, mapped)) + merged.sort(key=lambda row: (row[0], row[1])) + + suite_meta = {entry["suite"]: entry for entry in previous_meta.get("suites", [])} + for entry in fresh_meta.get("suites", []): + suite_meta[entry["suite"]] = entry + counts: dict[str, list[int]] = {} + for suite, test, status, _, _ in merged: + tally = counts.setdefault(suite, [0, 0]) + if test != SETUP: + tally[0] += 1 + tally[1] += status != "complete" + suites = [] + for suite in sorted(counts): + entry = dict(suite_meta.get(suite, {"suite": suite, "exit": 0})) + entry["tests"], entry["incomplete"] = counts[suite] + suites.append(entry) + + os.makedirs(args.out, exist_ok=True) + with open(os.path.join(args.out, "functions.tsv"), "w", encoding="utf-8") as handle: + handle.write("".join(line + "\n" for line in fresh_lines)) + with open(os.path.join(args.out, "tests.tsv"), "w", encoding="utf-8") as handle: + for suite, test, status, reason, ids in merged: + handle.write(f"{suite}:{test}\t{status}\t{reason}\t{' '.join(map(str, ids))}\n") + meta = dict(fresh_meta) + meta.update({"commit": args.commit, "tests": sum(c[0] for c in counts.values()), + "incomplete": sum(c[1] for c in counts.values()), "suites": suites}) + with open(os.path.join(args.out, "meta.json"), "w", encoding="utf-8") as handle: + json.dump(meta, handle, indent=1) + handle.write("\n") + print(f"merged: {len(rerun)} suite(s) re-run, " + f"{len(counts) - len(rerun & set(counts))} carried, {len(merged)} rows") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test.sh b/scripts/test.sh index 7bc7f4ea5..91594e581 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -370,6 +370,11 @@ bash "$ROOT/tests/test_test_impact_predict.sh" # child, an unreadable profile, a failed suite) has to come out `incomplete`. echo "=== Step 0z6: per-test coverage map builder contract ===" bash "$ROOT/tests/test_coverage_map.sh" +# Step 0z7: a push's incremental coverage map carries the suites it did not +# re-run onto the new image's function table; a wrong remap would hand the PR +# selection a map that claims tests ran functions they never touched. +echo "=== Step 0z7: incremental coverage map merge contract ===" +bash "$ROOT/tests/test_coverage_merge.sh" if [ "$CONTRACTS_ONLY" -eq 1 ]; then echo "=== test.sh: contracts-only — every contract step passed ===" diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index 01076ae48..cb384cd20 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -178,6 +178,10 @@ cbm_daemon_process_role_t cbm_daemon_process_role(int argc, char *const argv[]) * project, so it needs no daemon. Listed here rather than routed through * the daemon so enrolling a root cannot depend on daemon state. */ "allow-root", + /* test-impact runs its own frozen index of a pinned tree in process + * (CI: publish the team artifact, select tests); it never reads or + * writes the daemon's projects, so it needs no daemon. */ + "test-impact", }; /* Stop at the first top-level mode token. Tool names, flag values, and JSON * following `cli` are opaque user input: a search query named "install" diff --git a/src/main.c b/src/main.c index fd3370c04..67c1212a4 100644 --- a/src/main.c +++ b/src/main.c @@ -33,6 +33,7 @@ #include "daemon/version_cohort.h" #include "mcp/mcp.h" #include "mcp/index_supervisor.h" +#include "mcp/test_impact_engine.h" #include "cli/cli.h" #include "cli/progress_sink.h" #include "foundation/constants.h" @@ -1280,6 +1281,10 @@ static int handle_subcommand(int argc, char **argv, cbm_project_lock_manager_t * cbm_mem_init(cbm_mem_ram_fraction_for_total(cbm_system_info().total_ram)); return cbm_cmd_hook_augment(argc - i - SKIP_ONE, argv + i + SKIP_ONE); } + if (strcmp(argv[i], "test-impact") == 0) { + cbm_mem_init(cbm_mem_ram_fraction_for_total(cbm_system_info().total_ram)); + return cbm_test_impact_cli(argc - i - SKIP_ONE, argv + i + SKIP_ONE); + } if (strcmp(argv[i], "install") == 0) { return cbm_cmd_install(argc - i - SKIP_ONE, argv + i + SKIP_ONE); } diff --git a/src/mcp/test_impact.h b/src/mcp/test_impact.h index 7153e0e8d..56cc012b9 100644 --- a/src/mcp/test_impact.h +++ b/src/mcp/test_impact.h @@ -546,6 +546,10 @@ void cbm_test_policy_free(cbm_test_policy_t *policy); const cbm_test_rule_t *cbm_test_policy_rules(const cbm_test_policy_t *policy, int *count); const cbm_test_lane_t *cbm_test_policy_lanes(const cbm_test_policy_t *policy, int *count); const char *cbm_test_policy_digest(const cbm_test_policy_t *policy); +/* test_impact.coverage.compatibility_paths in config order: the files besides + * the measured code whose bytes a coverage map depends on (harness, runner, + * spawn helpers, build flags). NULL with *count 0 when none are configured. */ +const char *const *cbm_test_policy_compatibility_paths(const cbm_test_policy_t *policy, int *count); /* Successful classification returns true and either the first matched rule * or NULL for no match. false is an evaluation error, NEVER permission to diff --git a/src/mcp/test_impact_artifact.c b/src/mcp/test_impact_artifact.c new file mode 100644 index 000000000..dbeb2808c --- /dev/null +++ b/src/mcp/test_impact_artifact.c @@ -0,0 +1,283 @@ +/* + * test_impact_artifact.c — receipt.json and the bundle digests. + */ +#include "mcp/test_impact_artifact.h" + +#include "foundation/compat_fs.h" +#include "foundation/mem_core.h" +#include "foundation/sha256.h" + +#include +#include +#include + +enum { TA_READ_CHUNK = 65536, TA_RECEIPT_MAX = 65536, TA_PATH_CAP = 4096 }; + +/* yyjson allocates through the memory core, so documents and written text + * are released with cbm_free like everything else here. */ +static void *ta_yy_malloc(void *ctx, size_t size) { + (void)ctx; + return cbm_alloc(CBM_MEM_CLASS_OTHER, size); +} + +static void *ta_yy_realloc(void *ctx, void *block, size_t old_size, size_t size) { + (void)ctx; + (void)old_size; + return cbm_realloc(CBM_MEM_CLASS_OTHER, block, size); +} + +static void ta_yy_free(void *ctx, void *block) { + (void)ctx; + cbm_free(CBM_MEM_CLASS_OTHER, block); +} + +static const yyjson_alc ta_yy_alc = {ta_yy_malloc, ta_yy_realloc, ta_yy_free, NULL}; + +static bool ta_hex(const char *text, size_t want_a, size_t want_b) { + size_t n = text ? strlen(text) : 0; + if (n != want_a && n != want_b) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!((text[i] >= '0' && text[i] <= '9') || (text[i] >= 'a' && text[i] <= 'f'))) { + return false; + } + } + return true; +} + +static void ta_hex_out(const uint8_t digest[CBM_SHA256_DIGEST_LEN], char hex[65]) { + static const char digits[] = "0123456789abcdef"; + for (int i = 0; i < CBM_SHA256_DIGEST_LEN; i++) { + hex[2 * i] = digits[digest[i] >> 4]; + hex[2 * i + 1] = digits[digest[i] & 15]; + } + hex[64] = '\0'; +} + +static bool ta_path(char *out, const char *dir, const char *name) { + int n = snprintf(out, TA_PATH_CAP, "%s/%s", dir, name); + return n > 0 && n < TA_PATH_CAP; +} + +/* A length-prefixed frame, so no field can run into the next. */ +static void ta_frame(cbm_sha256_ctx *ctx, const void *bytes, uint64_t length) { + uint8_t prefix[8]; + for (int i = 0; i < 8; i++) { + prefix[i] = (uint8_t)(length >> (56 - 8 * i)); + } + cbm_sha256_update(ctx, prefix, sizeof(prefix)); + if (length) { + cbm_sha256_update(ctx, bytes, (size_t)length); + } +} + +/* Feeds a file's exact bytes as one frame; false on a read error. + * *absent: the file does not exist (a frame-free outcome the caller encodes). */ +static bool ta_feed_file(cbm_sha256_ctx *ctx, const char *path, bool framed, bool *absent) { + *absent = false; + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + cbm_path_info_t info; + *absent = cbm_path_info_utf8(path, &info) != 0; + return *absent; + } + unsigned char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, TA_READ_CHUNK); + if (!buf) { + (void)fclose(f); + return false; + } + /* Size first, so a framed file's prefix is its real length. */ + uint64_t total = 0; + size_t got; + while ((got = fread(buf, 1, TA_READ_CHUNK, f)) > 0) { + total += got; + } + bool ok = !ferror(f) && fseek(f, 0, SEEK_SET) == 0; + if (ok && framed) { + ta_frame(ctx, NULL, 0); /* domain: a present file */ + uint8_t prefix[8]; + for (int i = 0; i < 8; i++) { + prefix[i] = (uint8_t)(total >> (56 - 8 * i)); + } + cbm_sha256_update(ctx, prefix, sizeof(prefix)); + } + uint64_t again = 0; + while (ok && (got = fread(buf, 1, TA_READ_CHUNK, f)) > 0) { + cbm_sha256_update(ctx, buf, got); + again += got; + } + ok = ok && !ferror(f) && again == total; + cbm_free(CBM_MEM_CLASS_OTHER, buf); + ok = fclose(f) == 0 && ok; + return ok; +} + +bool cbm_ti_sha256_file(const char *path, char hex[65]) { + hex[0] = '\0'; + cbm_sha256_ctx ctx; + cbm_sha256_init(&ctx); + bool absent = false; + if (!path || !ta_feed_file(&ctx, path, false, &absent) || absent) { + return false; + } + uint8_t digest[CBM_SHA256_DIGEST_LEN]; + cbm_sha256_final(&ctx, digest); + ta_hex_out(digest, hex); + return true; +} + +bool cbm_ti_compatibility_digest(const char *root, const char *const *paths, int path_count, + const char *platform, char hex[65]) { + hex[0] = '\0'; + if (!root || !platform || path_count < 0 || (path_count && !paths)) { + return false; + } + static const char domain[] = "cbm.test_impact.compatibility.v1"; + cbm_sha256_ctx ctx; + cbm_sha256_init(&ctx); + ta_frame(&ctx, domain, sizeof(domain) - 1); + ta_frame(&ctx, platform, strlen(platform)); + for (int i = 0; i < path_count; i++) { + char path[TA_PATH_CAP]; + if (!paths[i] || !*paths[i] || !ta_path(path, root, paths[i])) { + return false; + } + ta_frame(&ctx, paths[i], strlen(paths[i])); + bool absent = false; + if (!ta_feed_file(&ctx, path, true, &absent)) { + return false; + } + if (absent) { + static const char marker[] = "absent"; + ta_frame(&ctx, marker, sizeof(marker) - 1); + } + } + uint8_t digest[CBM_SHA256_DIGEST_LEN]; + cbm_sha256_final(&ctx, digest); + ta_hex_out(digest, hex); + return true; +} + +static bool ta_copy_string(yyjson_val *root, const char *key, char *out, size_t cap) { + yyjson_val *v = yyjson_obj_get(root, key); + const char *s = yyjson_is_str(v) ? yyjson_get_str(v) : NULL; + if (!s || strlen(s) >= cap) { + return false; + } + memcpy(out, s, strlen(s) + 1); + return true; +} + +cbm_ti_receipt_status_t cbm_ti_receipt_read(const char *bundle_dir, cbm_ti_receipt_t *out) { + if (!out) { + return CBM_TI_RECEIPT_INVALID; + } + memset(out, 0, sizeof(*out)); + char path[TA_PATH_CAP]; + if (!bundle_dir || !ta_path(path, bundle_dir, CBM_TI_RECEIPT_FILE)) { + return CBM_TI_RECEIPT_INVALID; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + cbm_path_info_t info; + return cbm_path_info_utf8(path, &info) != 0 ? CBM_TI_RECEIPT_ABSENT : CBM_TI_RECEIPT_IO; + } + char *text = cbm_alloc(CBM_MEM_CLASS_OTHER, TA_RECEIPT_MAX + 1); + size_t len = text ? fread(text, 1, TA_RECEIPT_MAX + 1, f) : 0; + bool read_ok = text && !ferror(f) && len <= TA_RECEIPT_MAX; + (void)fclose(f); + if (!read_ok) { + cbm_free(CBM_MEM_CLASS_OTHER, text); + return text ? CBM_TI_RECEIPT_INVALID : CBM_TI_RECEIPT_IO; + } + yyjson_doc *doc = yyjson_read_opts(text, len, 0, &ta_yy_alc, NULL); + cbm_free(CBM_MEM_CLASS_OTHER, text); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + cbm_ti_receipt_t r = {0}; + char schema[64] = ""; + bool ok = yyjson_is_obj(root) && ta_copy_string(root, "schema", schema, sizeof(schema)) && + strcmp(schema, CBM_TI_RECEIPT_SCHEMA) == 0 && + ta_copy_string(root, "commit", r.commit, sizeof(r.commit)) && + ta_hex(r.commit, 40, 64) && + ta_copy_string(root, "graph_sha256", r.graph_sha256, sizeof(r.graph_sha256)) && + ta_hex(r.graph_sha256, 64, 64) && + ta_copy_string(root, "graph_content_sha256", r.graph_content_sha256, + sizeof(r.graph_content_sha256)) && + ta_hex(r.graph_content_sha256, 64, 64) && + ta_copy_string(root, "graph_topology_sha256", r.graph_topology_sha256, + sizeof(r.graph_topology_sha256)) && + ta_hex(r.graph_topology_sha256, 64, 64) && + ta_copy_string(root, "platform", r.platform, sizeof(r.platform)); + yyjson_val *coverage = ok ? yyjson_obj_get(root, "coverage") : NULL; + if (ok && coverage && !yyjson_is_null(coverage)) { + yyjson_val *oldest = yyjson_obj_get(coverage, "oldest_observation_at"); + r.has_coverage = true; + ok = yyjson_is_obj(coverage) && + ta_copy_string(coverage, "functions_sha256", r.functions_sha256, 65) && + ta_hex(r.functions_sha256, 64, 64) && + ta_copy_string(coverage, "tests_sha256", r.tests_sha256, 65) && + ta_hex(r.tests_sha256, 64, 64) && + ta_copy_string(coverage, "metadata_sha256", r.metadata_sha256, 65) && + ta_hex(r.metadata_sha256, 64, 64) && + ta_copy_string(coverage, "compatibility_sha256", r.compatibility_sha256, 65) && + ta_hex(r.compatibility_sha256, 64, 64) && yyjson_is_int(oldest) && + yyjson_get_sint(oldest) > 0; + r.oldest_observation_at = ok ? yyjson_get_sint(oldest) : 0; + } + yyjson_doc_free(doc); + if (!ok) { + return CBM_TI_RECEIPT_INVALID; + } + *out = r; + return CBM_TI_RECEIPT_OK; +} + +bool cbm_ti_receipt_write(const char *bundle_dir, const cbm_ti_receipt_t *r) { + char path[TA_PATH_CAP]; + char tmp[TA_PATH_CAP]; + if (!bundle_dir || !r || !ta_path(path, bundle_dir, CBM_TI_RECEIPT_FILE) || + !ta_path(tmp, bundle_dir, CBM_TI_RECEIPT_FILE ".tmp")) { + return false; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(&ta_yy_alc); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + if (!root) { + yyjson_mut_doc_free(doc); + return false; + } + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "schema", CBM_TI_RECEIPT_SCHEMA); + yyjson_mut_obj_add_str(doc, root, "commit", r->commit); + yyjson_mut_obj_add_str(doc, root, "graph_sha256", r->graph_sha256); + yyjson_mut_obj_add_str(doc, root, "graph_content_sha256", r->graph_content_sha256); + yyjson_mut_obj_add_str(doc, root, "graph_topology_sha256", r->graph_topology_sha256); + yyjson_mut_obj_add_str(doc, root, "platform", r->platform); + if (r->has_coverage) { + yyjson_mut_val *coverage = yyjson_mut_obj_add_obj(doc, root, "coverage"); + yyjson_mut_obj_add_str(doc, coverage, "functions_sha256", r->functions_sha256); + yyjson_mut_obj_add_str(doc, coverage, "tests_sha256", r->tests_sha256); + yyjson_mut_obj_add_str(doc, coverage, "metadata_sha256", r->metadata_sha256); + yyjson_mut_obj_add_str(doc, coverage, "compatibility_sha256", r->compatibility_sha256); + yyjson_mut_obj_add_int(doc, coverage, "oldest_observation_at", r->oldest_observation_at); + } else { + yyjson_mut_obj_add_null(doc, root, "coverage"); + } + size_t len = 0; + char *json = yyjson_mut_write_opts(doc, YYJSON_WRITE_PRETTY, &ta_yy_alc, &len, NULL); + yyjson_mut_doc_free(doc); + if (!json) { + return false; + } + FILE *f = cbm_fopen(tmp, "wb"); + bool ok = f && fwrite(json, 1, len, f) == len && fputc('\n', f) != EOF; + cbm_free(CBM_MEM_CLASS_OTHER, json); + if (f) { + ok = fclose(f) == 0 && ok; + } + ok = ok && cbm_rename_replace(tmp, path) == 0; + if (!ok) { + (void)cbm_unlink(tmp); + } + return ok; +} diff --git a/src/mcp/test_impact_artifact.h b/src/mcp/test_impact_artifact.h new file mode 100644 index 000000000..aa11b5a61 --- /dev/null +++ b/src/mcp/test_impact_artifact.h @@ -0,0 +1,75 @@ +/* + * test_impact_artifact.h — The team artifact bundle of one commit. + * + * A bundle travels from the main-branch producer to PR selection (never + * committed). Layout under its directory: + * + * receipt.json this module; the bundle's authority + * .codebase-memory/graph.db.zst the commit's frozen graph (artifact.h) + * .codebase-memory/artifact.json + * coverage/functions.tsv optional coverage map, format 1 + * coverage/tests.tsv + * coverage/meta.json + * + * The receipt binds every part by SHA-256: the graph by the digest of the + * graph as an importer sees it (cbm_store_graph_digest of the imported + * database), the map by the exact bytes of its three files. A digest is + * content binding only; where a bundle came from is established by whoever + * fetched it, never by anything inside it. + */ +#ifndef CBM_TEST_IMPACT_ARTIFACT_H +#define CBM_TEST_IMPACT_ARTIFACT_H + +#include +#include +#include + +#define CBM_TI_RECEIPT_FILE "receipt.json" +#define CBM_TI_RECEIPT_SCHEMA "cbm.test_impact.artifact.v1" +#define CBM_TI_COVERAGE_DIR "coverage" + +typedef struct { + char commit[65]; /* 40 or 64 lowercase hex */ + char graph_sha256[65]; /* 64 lowercase hex */ + /* cbm_store_graph_content_digest of the same graph: equal for equal graphs + * however they were built (the weekly incremental-equals-full check). */ + char graph_content_sha256[65]; + /* cbm_store_graph_topology_digest: what selection reads. The weekly check + * fails when an incremental graph's topology differs from the full one. */ + char graph_topology_sha256[65]; + bool has_coverage; + char functions_sha256[65]; + char tests_sha256[65]; + char metadata_sha256[65]; + char compatibility_sha256[65]; + /* Oldest retained coverage observation (seconds since the epoch). A full + * refresh sets it; an incremental publish carries it forward unchanged. */ + int64_t oldest_observation_at; + char platform[128]; /* producer's platform/toolchain label, informational */ +} cbm_ti_receipt_t; + +typedef enum { + CBM_TI_RECEIPT_OK = 0, + CBM_TI_RECEIPT_ABSENT, /* no receipt.json */ + CBM_TI_RECEIPT_INVALID, /* unreadable shape, unknown schema, a malformed field */ + CBM_TI_RECEIPT_IO, +} cbm_ti_receipt_status_t; + +/* Strict: every field present and well formed, no unknown schema. Clears *out + * first; on any non-OK status *out stays zero. */ +cbm_ti_receipt_status_t cbm_ti_receipt_read(const char *bundle_dir, cbm_ti_receipt_t *out); + +/* Writes receipt.json atomically (temporary file + rename). */ +bool cbm_ti_receipt_write(const char *bundle_dir, const cbm_ti_receipt_t *receipt); + +/* Lowercase SHA-256 hex of a file's exact bytes. false when unreadable. */ +bool cbm_ti_sha256_file(const char *path, char hex[65]); + +/* Versioned digest of what coverage depends on beyond the code it measures: + * the platform/toolchain label and the exact bytes of each listed path under + * root (an absent path is encoded as absent, so adding one changes it). + * Producer and consumer call this same function; equality is the check. */ +bool cbm_ti_compatibility_digest(const char *root, const char *const *paths, int path_count, + const char *platform, char hex[65]); + +#endif /* CBM_TEST_IMPACT_ARTIFACT_H */ diff --git a/src/mcp/test_impact_cli.c b/src/mcp/test_impact_cli.c new file mode 100644 index 000000000..4106d89db --- /dev/null +++ b/src/mcp/test_impact_cli.c @@ -0,0 +1,185 @@ +/* + * test_impact_cli.c — `codebase-memory-mcp test-impact `. + * + * The CI surface of the test-impact engine, in process (no daemon): + * publish builds the team artifact bundle of the checked-out commit; + * select answers which tests to run for the change against a base ref. + * + * A bundle takes part in a selection only with --artifact-verified, which the + * CI job passes after it fetched the bundle from this repository's + * main-branch producer run whose head is --artifact-commit. Nothing inside a + * bundle can assert that. + */ +#include "mcp/test_impact_engine.h" + +#include "foundation/compat_fs.h" +#include "foundation/platform.h" + +#include +#include +#include + +enum { TIC_DEFAULT_DEADLINE_S = 1800 }; + +typedef struct { + const char *name; + const char **value; /* NULL: a flag */ + bool *flag; +} tic_option_t; + +static bool tic_parse(int argc, char **argv, const tic_option_t *options, size_t count) { + for (int i = 0; i < argc; i++) { + const tic_option_t *match = NULL; + for (size_t o = 0; o < count; o++) { + if (strcmp(argv[i], options[o].name) == 0) { + match = &options[o]; + } + } + if (!match) { + (void)fprintf(stderr, "test-impact: unknown argument %s\n", argv[i]); + return false; + } + if (match->flag) { + *match->flag = true; + continue; + } + if (i + 1 >= argc) { + (void)fprintf(stderr, "test-impact: %s needs a value\n", argv[i]); + return false; + } + *match->value = argv[++i]; + } + return true; +} + +static int64_t tic_number(const char *text, int64_t fallback) { + if (!text || !*text) { + return fallback; + } + char *end = NULL; + long long value = strtoll(text, &end, 10); + return end && !*end && value > 0 ? (int64_t)value : -1; +} + +/* /test-impact unless --work names another private parent. */ +static const char *tic_work(const char *requested, char *buf, size_t cap) { + if (requested) { + return requested; + } + snprintf(buf, cap, "%s/test-impact", cbm_resolve_cache_dir()); + return cbm_mkdir_p(buf, 0700) ? buf : NULL; +} + +static int tic_usage(void) { + (void)fprintf(stderr, + "usage: codebase-memory-mcp test-impact publish --repo DIR --out DIR\n" + " [--previous BUNDLE] [--coverage DIR --observed-at EPOCH]\n" + " [--platform LABEL] [--config FILE] [--work DIR] [--deadline-s N]\n" + " codebase-memory-mcp test-impact select --repo DIR --base REF\n" + " [--config FILE] [--artifact BUNDLE --artifact-commit SHA\n" + " --artifact-verified] [--platform LABEL] [--work DIR]\n" + " [--deadline-s N]\n"); + return 2; +} + +static int tic_publish(int argc, char **argv) { + const char *repo = NULL, *out = NULL, *previous = NULL, *coverage = NULL; + const char *observed = NULL, *platform = NULL, *work = NULL, *deadline = NULL; + const char *config = NULL; + const tic_option_t options[] = {{"--config", &config, NULL}, + {"--repo", &repo, NULL}, + {"--out", &out, NULL}, + {"--previous", &previous, NULL}, + {"--coverage", &coverage, NULL}, + {"--observed-at", &observed, NULL}, + {"--platform", &platform, NULL}, + {"--work", &work, NULL}, + {"--deadline-s", &deadline, NULL}}; + if (!tic_parse(argc, argv, options, sizeof(options) / sizeof(options[0])) || !repo || !out) { + return tic_usage(); + } + int64_t observed_at = tic_number(observed, 0); + int64_t seconds = tic_number(deadline, TIC_DEFAULT_DEADLINE_S); + char work_buf[4096]; + const char *work_parent = tic_work(work, work_buf, sizeof(work_buf)); + if (observed_at < 0 || seconds < 0 || !work_parent) { + return tic_usage(); + } + cbm_test_impact_publish_t request = {.repo_root = repo, + .work_parent = work_parent, + .out_dir = out, + .previous_dir = previous, + .coverage_dir = coverage, + .observed_at = observed_at, + .platform = platform, + .config_path = config, + .deadline_ms = cbm_now_ms() + (uint64_t)seconds * 1000ULL}; + char diagnostic[512] = ""; + cbm_test_impact_status_t status = + cbm_test_impact_publish(&request, diagnostic, sizeof(diagnostic)); + (void)fprintf(stderr, "test-impact publish: status %d%s%s\n", (int)status, + diagnostic[0] ? ": " : "", diagnostic); + return status == CBM_TEST_IMPACT_OK ? 0 : 1; +} + +static int tic_select(int argc, char **argv) { + const char *repo = NULL, *base = NULL, *config = NULL, *artifact = NULL; + const char *artifact_commit = NULL, *platform = NULL, *work = NULL, *deadline = NULL; + bool verified = false; + const tic_option_t options[] = {{"--repo", &repo, NULL}, + {"--base", &base, NULL}, + {"--config", &config, NULL}, + {"--artifact", &artifact, NULL}, + {"--artifact-commit", &artifact_commit, NULL}, + {"--artifact-verified", NULL, &verified}, + {"--platform", &platform, NULL}, + {"--work", &work, NULL}, + {"--deadline-s", &deadline, NULL}}; + if (!tic_parse(argc, argv, options, sizeof(options) / sizeof(options[0])) || !repo || !base) { + return tic_usage(); + } + int64_t seconds = tic_number(deadline, TIC_DEFAULT_DEADLINE_S); + char work_buf[4096]; + const char *work_parent = tic_work(work, work_buf, sizeof(work_buf)); + if (seconds < 0 || !work_parent) { + return tic_usage(); + } + cbm_test_impact_request_t request = {.repo_root = repo, + .base_ref = base, + .work_parent = work_parent, + .config_path = config, + .deadline_ms = cbm_now_ms() + (uint64_t)seconds * 1000ULL, + .artifact_dir = artifact, + .artifact_commit = artifact_commit, + .artifact_verified = verified, + .platform = platform}; + cbm_test_result_t *result = NULL; + char diagnostic[512] = ""; + cbm_test_impact_status_t status = + cbm_test_impact_run(&request, &result, diagnostic, sizeof(diagnostic)); + if (diagnostic[0]) { + (void)fprintf(stderr, "test-impact select: %s\n", diagnostic); + } + if (status != CBM_TEST_IMPACT_OK) { + (void)fprintf(stderr, "test-impact select: status %d\n", (int)status); + return 1; + } + size_t length = 0; + const char *json = cbm_test_result_json(result, &length); + bool ok = json && fwrite(json, 1, length, stdout) == length && fputc('\n', stdout) != EOF; + cbm_test_result_free(result); + return ok ? 0 : 1; +} + +int cbm_test_impact_cli(int argc, char **argv) { + if (argc < 1) { + return tic_usage(); + } + if (strcmp(argv[0], "publish") == 0) { + return tic_publish(argc - 1, argv + 1); + } + if (strcmp(argv[0], "select") == 0) { + return tic_select(argc - 1, argv + 1); + } + return tic_usage(); +} diff --git a/src/mcp/test_impact_engine.c b/src/mcp/test_impact_engine.c index 39aece400..391a6d6d7 100644 --- a/src/mcp/test_impact_engine.c +++ b/src/mcp/test_impact_engine.c @@ -43,7 +43,7 @@ bool te_oom(te_ctx_t *c) { return false; } -static void te_note(te_ctx_t *c, const char *text) { +void te_note(te_ctx_t *c, const char *text) { if (c->diagnostic && c->diagnostic_len && !c->diagnostic[0] && text) { snprintf(c->diagnostic, c->diagnostic_len, "%s", text); } @@ -103,7 +103,7 @@ bool cbm_test_impact_find_git(char *out, size_t out_len) { return false; } -static bool te_git(te_ctx_t *c) { +bool te_git(te_ctx_t *c) { char git[4096]; if (!cbm_test_impact_find_git(git, sizeof(git))) { return te_fallback(c, CBM_TEST_RESULT_FALLBACK_GIT_UNAVAILABLE, "no git on PATH") && false; @@ -139,7 +139,7 @@ static bool te_git(te_ctx_t *c) { /* ── Query-time configuration: the merge base's (M-4) ────────────── */ -static bool te_config(te_ctx_t *c) { +bool te_config(te_ctx_t *c) { const char *path = c->rq->config_path; char file[4200] = ""; bool optional = false; @@ -391,7 +391,11 @@ static bool te_select(te_ctx_t *c) { .diff_complete = c->diff_complete, .inventory_complete = c->snapshot_ok, .static_complete = c->static_complete, - .coverage_admitted = false}; + .coverage = c->coverage_admitted ? c->coverage : NULL, + .changed_function_ids = c->cov_ids, + .changed_function_count = c->cov_count, + .coverage_admitted = c->coverage_admitted, + .coverage_changes_complete = c->cov_complete}; c->selection = cbm_test_select(&input); if (!c->selection) { return te_fallback(c, CBM_TEST_RESULT_FALLBACK_SELECTION_UNAVAILABLE, @@ -487,7 +491,7 @@ static void te_receipt(te_ctx_t *c, cbm_test_result_receipt_t *r, *graph_reason = CBM_TEST_RESULT_EVIDENCE_DIAGNOSTICS_INCOMPLETE; r->graph.rejection_reasons = (cbm_test_result_evidence_reasons_t){graph_reason, 1}; } - r->coverage.state = CBM_TEST_RESULT_COVERAGE_UNAVAILABLE; + te_artifact_receipt(c, &r->coverage, te_oid); } static cbm_test_impact_status_t te_answer(te_ctx_t *c, cbm_test_result_t **out) { @@ -548,6 +552,8 @@ static cbm_test_impact_status_t te_answer(te_ctx_t *c, cbm_test_result_t **out) static void te_free(te_ctx_t *c) { cbm_test_selection_free(c->selection); + cbm_coverage_map_free(c->coverage); + cbm_free(CBM_MEM_CLASS_OTHER, c->coverage_meta); cbm_ti_seeds_free(c->seeds); if (c->scope) { (void)cbm_store_read_scope_close(c->scope); @@ -584,10 +590,11 @@ static void te_steps(te_ctx_t *c) { if (!te_classify_rules(c, &need_snapshot, &unmatched) || !need_snapshot) { return; } - if (!te_snapshot(c) || !te_classify_unmatched(c, unmatched) || c->fallback_count) { + if (!te_artifact_graph(c) || !te_snapshot(c) || !te_artifact_coverage(c) || + !te_classify_unmatched(c, unmatched) || c->fallback_count) { return; } - if (!te_open_scope(c) || !te_seed(c) || !te_reach(c)) { + if (!te_open_scope(c) || !te_seed(c) || !te_reach(c) || !te_coverage_ids(c)) { return; } (void)te_select(c); @@ -648,3 +655,49 @@ cbm_test_impact_status_t cbm_test_impact_run(const cbm_test_impact_request_t *re (void)cbm_thread_join(&thread); return job.status; } + +typedef struct { + const cbm_test_impact_publish_t *request; + char *diagnostic; + size_t diagnostic_len; + cbm_test_impact_status_t status; +} te_publish_job_t; + +static void *te_publish_main(void *arg) { + te_publish_job_t *job = arg; + const cbm_test_impact_publish_t *p = job->request; + /* The steps read a selection request; publishing is the empty change of + * HEAD against itself. */ + cbm_test_impact_request_t rq = {.repo_root = p->repo_root, + .base_ref = "HEAD", + .work_parent = p->work_parent, + .config_path = p->config_path, + .deadline_ms = p->deadline_ms}; + te_ctx_t c = {.rq = &rq, .diagnostic = job->diagnostic, .diagnostic_len = job->diagnostic_len}; + cbm_arena_init(&c.arena); + bool ok = te_publish(&c, p); + job->status = c.oom ? CBM_TEST_IMPACT_OOM : ok ? CBM_TEST_IMPACT_OK : CBM_TEST_IMPACT_INVALID; + te_free(&c); + return NULL; +} + +cbm_test_impact_status_t cbm_test_impact_publish(const cbm_test_impact_publish_t *request, + char *diagnostic, size_t diagnostic_len) { + if (diagnostic && diagnostic_len) { + diagnostic[0] = '\0'; + } + if (!request || !request->repo_root || !request->work_parent || !request->out_dir || + !request->deadline_ms) { + return CBM_TEST_IMPACT_INVALID; + } + te_publish_job_t job = {.request = request, + .diagnostic = diagnostic, + .diagnostic_len = diagnostic_len, + .status = CBM_TEST_IMPACT_OOM}; + cbm_thread_t thread; /* a whole frozen index: the default pipeline stack */ + if (cbm_thread_create(&thread, 0, te_publish_main, &job) != 0) { + return CBM_TEST_IMPACT_OOM; + } + (void)cbm_thread_join(&thread); + return job.status; +} diff --git a/src/mcp/test_impact_engine.h b/src/mcp/test_impact_engine.h index 89cc50050..6ce3e51fb 100644 --- a/src/mcp/test_impact_engine.h +++ b/src/mcp/test_impact_engine.h @@ -33,6 +33,20 @@ typedef struct { * (a change cannot loosen its own selection). Otherwise that file. */ const char *config_path; uint64_t deadline_ms; /* absolute cbm_now_ms() deadline */ + + /* Optional team artifact bundle (test_impact_artifact.h) of the merge + * base. It takes part ONLY with artifact_verified: the caller fetched it + * from this repository's main-branch producer whose head is + * artifact_commit. Its graph then becomes the build's incremental base + * and its coverage map may be admitted (receipt policy, age, ancestry, + * compatibility). Without it, or on any doubt, the graph is built from + * scratch and no coverage is admitted: every suite runs whole. */ + const char *artifact_dir; + const char *artifact_commit; + bool artifact_verified; + /* Platform/toolchain label of the leg the answer is for; part of the + * coverage compatibility digest, so a map narrows only its own platform. */ + const char *platform; } cbm_test_impact_request_t; typedef enum { @@ -52,4 +66,32 @@ cbm_test_impact_status_t cbm_test_impact_run(const cbm_test_impact_request_t *re * executable `git` (`git.exe` on Windows) on PATH. false when none. */ bool cbm_test_impact_find_git(char *out, size_t out_len); +/* Builds the team artifact bundle of the checked-out commit of repo_root: + * the commit's frozen graph (incrementally from previous_dir's graph when that + * bundle is given and verifies), the coverage map of this commit when given, + * and receipt.json binding all of it. out_dir must not exist. */ +typedef struct { + const char *repo_root; + const char *work_parent; + const char *out_dir; + const char *previous_dir; /* optional: a bundle of an ancestor */ + const char *coverage_dir; /* optional: functions.tsv, tests.tsv, meta.json */ + /* When the map was recorded (seconds since the epoch). 0 carries the + * previous bundle's oldest observation forward (an incremental merge never + * advances it). */ + int64_t observed_at; + const char *platform; + /* NULL: the commit's own .codebase-memory.json. Otherwise that file, as + * for a selection (its coverage.compatibility_paths go into the receipt). */ + const char *config_path; + uint64_t deadline_ms; +} cbm_test_impact_publish_t; + +cbm_test_impact_status_t cbm_test_impact_publish(const cbm_test_impact_publish_t *request, + char *diagnostic, size_t diagnostic_len); + +/* `codebase-memory-mcp test-impact ...` (test_impact_cli.c); + * argv starts at the subcommand. The process exit code. */ +int cbm_test_impact_cli(int argc, char **argv); + #endif /* CBM_TEST_IMPACT_ENGINE_H */ diff --git a/src/mcp/test_impact_engine_artifact.c b/src/mcp/test_impact_engine_artifact.c new file mode 100644 index 000000000..dee209b89 --- /dev/null +++ b/src/mcp/test_impact_engine_artifact.c @@ -0,0 +1,646 @@ +/* + * test_impact_engine_artifact.c — the team artifact on both ends. + * + * Producer (cbm_test_impact_publish): the main branch builds the bundle of a + * commit — the commit's frozen graph, incrementally from the previous bundle's + * graph when that bundle verifies, its coverage map, and receipt.json. + * + * Consumer (te_artifact_graph, te_artifact_coverage, te_coverage_ids): a PR + * selection given a bundle the caller verified uses the bundle's graph as the + * frozen build's incremental base and may admit its coverage map. Every doubt + * ends in the conservative answer: the graph built from scratch, the map not + * admitted, every suite whole. + */ +#include "mcp/test_impact_engine_internal.h" + +#include "foundation/compat_fs.h" +#include "foundation/mem_core.h" +#include "foundation/platform.h" +#include "mcp/test_impact_artifact.h" +#include "pipeline/artifact.h" +#include "store/store_graph_digest.h" + +#include +#include +#include + +enum { TA_FILE_MAX = 512 << 20 }; + +static void ta_hex(const unsigned char bytes[32], char hex[65]) { + static const char digits[] = "0123456789abcdef"; + for (int i = 0; i < 32; i++) { + hex[2 * i] = digits[bytes[i] >> 4]; + hex[2 * i + 1] = digits[bytes[i] & 15]; + } + hex[64] = '\0'; +} + +static bool ta_unhex(const char *hex, unsigned char bytes[32]) { + if (!hex || strlen(hex) != 64) { + return false; + } + for (int i = 0; i < 64; i++) { + char ch = hex[i]; + int v = ch >= '0' && ch <= '9' ? ch - '0' : ch >= 'a' && ch <= 'f' ? ch - 'a' + 10 : -1; + if (v < 0) { + return false; + } + bytes[i / 2] = (unsigned char)(i % 2 ? (bytes[i / 2] | v) : (v << 4)); + } + return true; +} + +/* The exact graph digest of a database: what an importer of it measures. */ +bool te_db_digest(const char *db_path, char hex[65]) { + hex[0] = '\0'; + cbm_store_t *store = cbm_store_open_path_query(db_path); + if (!store) { + return false; + } + cbm_store_read_scope_t *scope = NULL; + bool ok = cbm_store_read_scope_open(store, NULL, NULL, &scope) == CBM_STORE_OK; + static const unsigned char project[] = "test-impact"; + cbm_store_graph_digest_limits_t limits = {.max_rows = (uint64_t)1 << 32, + .max_framed_bytes = (uint64_t)1 << 40}; + cbm_store_graph_digest_t digest; + ok = ok && cbm_store_graph_digest(scope, project, sizeof(project) - 1, &limits, &digest) == + CBM_STORE_GRAPH_DIGEST_OK; + if (scope && cbm_store_read_scope_close(scope) != CBM_STORE_OK) { + ok = false; + } + cbm_store_close(store); + if (ok) { + ta_hex(digest.sha256, hex); + } + return ok; +} + +/* The graph-content digest (store_content_digest.c): equal for equal graphs + * however they were built; the weekly incremental-equals-full check. */ +static bool te_db_content_digest(const char *db_path, char content[65], char topology[65]) { + content[0] = '\0'; + topology[0] = '\0'; + cbm_store_t *store = cbm_store_open_path_query(db_path); + unsigned char a[32]; + unsigned char b[32]; + bool ok = store && cbm_store_graph_content_digest(store, "test-impact", a) == CBM_STORE_OK && + cbm_store_graph_topology_digest(store, "test-impact", b) == CBM_STORE_OK; + if (store) { + cbm_store_close(store); + } + if (ok) { + ta_hex(a, content); + ta_hex(b, topology); + } + return ok; +} + +/* Imports a bundle's graph to db_path; true only when it is the graph the + * receipt names. */ +static bool te_bundle_import(const char *bundle, const cbm_ti_receipt_t *receipt, + const char *db_path, const char **why) { + if (cbm_artifact_import(bundle, db_path) != 0) { + *why = "the bundle's graph could not be imported"; + return false; + } + char measured[65]; + if (!te_db_digest(db_path, measured)) { + *why = "the bundle's imported graph could not be digested"; + return false; + } + if (strcmp(measured, receipt->graph_sha256) != 0) { + *why = "the bundle's imported graph is not the graph its receipt names"; + return false; + } + return true; +} + +static bool te_read_file(const char *path, char **out, size_t *len) { + *out = NULL; + *len = 0; + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return false; + } + size_t cap = 65536; + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, cap); + size_t n = 0; + bool ok = buf != NULL; + while (ok) { + if (n == cap) { + if (cap >= TA_FILE_MAX) { + ok = false; + break; + } + char *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, buf, cap * 2); + if (!grown) { + ok = false; + break; + } + buf = grown; + cap *= 2; + } + size_t got = fread(buf + n, 1, cap - n, f); + n += got; + if (got == 0) { + ok = !ferror(f); + break; + } + } + (void)fclose(f); + if (!ok) { + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return false; + } + *out = buf; + *len = n; + return true; +} + +/* The coverage map of a bundle (or any directory with the three files). */ +static bool te_load_map(const char *dir, cbm_coverage_map_t **map, char **meta, size_t *meta_len) { + char path[4200]; + char *functions = NULL; + char *tests = NULL; + size_t functions_len = 0; + size_t tests_len = 0; + *map = NULL; + *meta = NULL; + *meta_len = 0; + snprintf(path, sizeof(path), "%s/functions.tsv", dir); + bool ok = te_read_file(path, &functions, &functions_len); + snprintf(path, sizeof(path), "%s/tests.tsv", dir); + ok = ok && te_read_file(path, &tests, &tests_len); + snprintf(path, sizeof(path), "%s/meta.json", dir); + ok = ok && te_read_file(path, meta, meta_len); + *map = ok ? cbm_coverage_map_parse(functions, functions_len, tests, tests_len) : NULL; + cbm_free(CBM_MEM_CLASS_OTHER, functions); + cbm_free(CBM_MEM_CLASS_OTHER, tests); + if (!*map) { + cbm_free(CBM_MEM_CLASS_OTHER, *meta); + *meta = NULL; + *meta_len = 0; + return false; + } + return true; +} + +/* The compatibility digest of this checkout's pinned tree for `platform`, + * over the configured compatibility paths. */ +static bool te_compatibility(te_ctx_t *c, const char *platform, char hex[65]) { + const cbm_pinned_tree_view_t *view = c->tree ? cbm_pinned_tree_view(c->tree) : NULL; + int count = 0; + const char *const *paths = cbm_test_policy_compatibility_paths(c->policy, &count); + return view && platform && *platform && + cbm_ti_compatibility_digest(view->root, paths, count, platform, hex); +} + +/* ── Consumer ──────────────────────────────────────────────────────── */ + +static void te_reject(te_ctx_t *c, cbm_test_result_evidence_reason_t reason, const char *why) { + if (c->coverage_reason_count < + (int)(sizeof(c->coverage_reasons) / sizeof(c->coverage_reasons[0]))) { + bool known = false; + for (int i = 0; i < c->coverage_reason_count; i++) { + known = known || c->coverage_reasons[i] == reason; + } + if (!known) { + c->coverage_reasons[c->coverage_reason_count++] = reason; + } + } + te_note(c, why); +} + +/* Before the snapshot: a verified bundle of the merge base becomes the frozen + * build's incremental base. */ +bool te_artifact_graph(te_ctx_t *c) { + const cbm_test_impact_request_t *rq = c->rq; + if (!rq->artifact_dir) { + return true; + } + c->artifact_requested = true; + if (!rq->artifact_verified || !rq->artifact_commit) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_SOURCE_UNVERIFIED, + "artifact: the caller did not verify the bundle's source; ignored"); + return true; + } + if (cbm_ti_receipt_read(rq->artifact_dir, &c->receipt) != CBM_TI_RECEIPT_OK) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_METADATA_INVALID, + "artifact: the bundle's receipt is missing or invalid"); + return true; + } + if (strcmp(c->receipt.commit, rq->artifact_commit) != 0) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_COMMIT_MISMATCH, + "artifact: the receipt names another commit than the verified source"); + return true; + } + /* v1 admits only the merge base's own bundle: an older ancestor's map + * cannot see what main changed since (recorded follow-up). */ + if (strcmp(c->receipt.commit, c->id->merge_base) != 0) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_ANCESTRY_UNPROVED, + "artifact: the bundle is not of the merge base"); + return true; + } + snprintf(c->base_db, sizeof(c->base_db), "%s/graph/base.db", c->work_dir); + const char *why = NULL; + if (!te_bundle_import(rq->artifact_dir, &c->receipt, c->base_db, &why)) { + (void)cbm_unlink(c->base_db); + (void)cbm_remove_db_sidecars(c->base_db); + c->base_db[0] = '\0'; + te_reject(c, CBM_TEST_RESULT_EVIDENCE_CONTENT_MISMATCH, why); + return true; + } + c->artifact_graph = true; + return true; +} + +/* After the snapshot: the receipt policy decides whether the map narrows. */ +bool te_artifact_coverage(te_ctx_t *c) { + const cbm_test_impact_request_t *rq = c->rq; + if (!c->artifact_graph) { + return true; + } + if (!c->receipt.has_coverage) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_PROVIDER_UNAVAILABLE, + "artifact: the bundle carries no coverage map"); + return true; + } + char dir[4200]; + snprintf(dir, sizeof(dir), "%s/%s", rq->artifact_dir, CBM_TI_COVERAGE_DIR); + if (!te_load_map(dir, &c->coverage, &c->coverage_meta, &c->coverage_meta_len)) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_FORMAT_UNSUPPORTED, + "artifact: the coverage map could not be read"); + return true; + } + char compatibility[65] = ""; + if (!te_compatibility(c, rq->platform, compatibility)) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_COMPATIBILITY_MISMATCH, + "artifact: no platform to judge the map's compatibility by"); + return true; + } + cbm_coverage_receipt_t receipt = {.commit = c->receipt.commit, + .functions_sha256 = c->receipt.functions_sha256, + .tests_sha256 = c->receipt.tests_sha256, + .metadata_sha256 = c->receipt.metadata_sha256, + .graph_sha256 = c->receipt.graph_sha256, + .compatibility_sha256 = c->receipt.compatibility_sha256, + .oldest_observation_at = c->receipt.oldest_observation_at}; + char measured[65]; + if (!te_db_digest(c->base_db, measured)) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_CONTENT_MISMATCH, + "artifact: the imported graph could not be digested"); + return true; + } + cbm_coverage_receipt_context_t context = {.source_verified = rq->artifact_verified, + .trusted_commit = rq->artifact_commit, + .ancestor_verified = + strcmp(c->receipt.commit, c->id->merge_base) == 0, + .ancestor_commit = c->receipt.commit, + .ancestor_merge_base = c->id->merge_base, + .merge_base = c->id->merge_base, + .graph_commit = c->receipt.commit, + .graph_sha256 = measured, + .compatibility_sha256 = compatibility, + .now = (int64_t)time(NULL)}; + unsigned reasons = cbm_coverage_map_check_receipt(c->coverage, c->coverage_meta, + c->coverage_meta_len, &receipt, &context); + static const struct { + unsigned bit; + cbm_test_result_evidence_reason_t reason; + } map[] = { + {CBM_COVERAGE_RECEIPT_INVALID, CBM_TEST_RESULT_EVIDENCE_METADATA_INVALID}, + {CBM_COVERAGE_RECEIPT_SOURCE, CBM_TEST_RESULT_EVIDENCE_SOURCE_UNVERIFIED}, + {CBM_COVERAGE_RECEIPT_ANCESTRY, CBM_TEST_RESULT_EVIDENCE_ANCESTRY_UNPROVED}, + {CBM_COVERAGE_RECEIPT_COMMIT, CBM_TEST_RESULT_EVIDENCE_COMMIT_MISMATCH}, + {CBM_COVERAGE_RECEIPT_CONTENT, CBM_TEST_RESULT_EVIDENCE_CONTENT_MISMATCH}, + {CBM_COVERAGE_RECEIPT_AGE, CBM_TEST_RESULT_EVIDENCE_TOO_OLD}, + {CBM_COVERAGE_RECEIPT_COMPATIBILITY, CBM_TEST_RESULT_EVIDENCE_COMPATIBILITY_MISMATCH}, + {CBM_COVERAGE_RECEIPT_METADATA, CBM_TEST_RESULT_EVIDENCE_METADATA_INVALID}}; + for (size_t i = 0; i < sizeof(map) / sizeof(map[0]); i++) { + if (reasons & map[i].bit) { + te_reject(c, map[i].reason, "artifact: the coverage receipt was refused"); + } + } + c->coverage_admitted = reasons == 0; + return true; +} + +/* Fills the coverage half of the answer's receipt. */ +void te_artifact_receipt(te_ctx_t *c, cbm_test_result_coverage_receipt_t *r, + cbm_test_result_oid_t (*oid)(const te_ctx_t *, const char *)) { + memset(r, 0, sizeof(*r)); + if (!c->artifact_requested) { + r->state = CBM_TEST_RESULT_COVERAGE_UNAVAILABLE; + return; + } + r->format = CBM_TEST_RESULT_COVERAGE_FORMAT_FUNCTIONS_V1; + r->artifact = (cbm_test_result_bytes_t){(const unsigned char *)CBM_TI_RECEIPT_SCHEMA, + sizeof(CBM_TI_RECEIPT_SCHEMA) - 1}; + if (c->receipt.commit[0]) { + r->commit = oid(c, c->receipt.commit); + } + struct { + const char *hex; + cbm_test_result_digest_t *out; + } digests[] = {{c->receipt.functions_sha256, &r->identities_sha256}, + {c->receipt.tests_sha256, &r->tests_sha256}, + {c->receipt.metadata_sha256, &r->metadata_sha256}, + {c->receipt.graph_sha256, &r->graph_sha256}, + {c->receipt.compatibility_sha256, &r->compatibility_sha256}}; + for (size_t i = 0; i < sizeof(digests) / sizeof(digests[0]); i++) { + digests[i].out->present = ta_unhex(digests[i].hex, digests[i].out->bytes); + if (!digests[i].out->present) { + memset(digests[i].out->bytes, 0, sizeof(digests[i].out->bytes)); + } + } + if (c->receipt.oldest_observation_at > 0) { + r->oldest_observation_at = + (cbm_test_result_timestamp_t){true, c->receipt.oldest_observation_at}; + } + if (c->coverage_admitted) { + r->state = CBM_TEST_RESULT_COVERAGE_ADMITTED; + return; + } + if (!c->coverage_reason_count) { + te_reject(c, CBM_TEST_RESULT_EVIDENCE_PROVIDER_UNAVAILABLE, NULL); + } + r->state = CBM_TEST_RESULT_COVERAGE_REJECTED; + r->rejection_reasons = + (cbm_test_result_evidence_reasons_t){c->coverage_reasons, (size_t)c->coverage_reason_count}; +} + +/* ── Coverage: the changed function set C_eff (R2.1) ──────────────── + * + * A seed the map knows contributes its own id: the map records every test + * that executed it, spawned children included, so its callers add nothing. + * A seed the map does not know (new code, a type, a macro, a field, code the + * map's image never compiled) contributes the nearest mapped functions on its + * static inbound closure: a test that executes any of them may now reach the + * change. Names deleted at head contribute every mapped function of that + * name. A store failure leaves the set incomplete, which runs everything. */ + +static const char *const te_cov_types[] = {"CALLS", "ASYNC_CALLS", "USAGE", "CALL_REFERENCE", + "READS"}; + +typedef struct { + te_ctx_t *c; + const cbm_coverage_map_t *map; + int *ids; + int count; + int cap; + int64_t *queue; + int queue_len; + int queue_cap; + CBMHashTable *visited; +} te_cov_t; + +static bool te_cov_add(te_cov_t *k, int id) { + if (k->count == k->cap) { + int cap = k->cap ? k->cap * 2 : 64; + int *grown = cbm_arena_alloc(&k->c->arena, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + if (k->count) { + memcpy(grown, k->ids, (size_t)k->count * sizeof(*grown)); + } + k->ids = grown; + k->cap = cap; + } + k->ids[k->count++] = id; + return true; +} + +static bool te_cov_push(te_cov_t *k, int64_t id) { + char key[24]; + snprintf(key, sizeof(key), "%lld", (long long)id); + if (cbm_ht_get(k->visited, key)) { + return true; + } + char *owned = cbm_arena_strdup(&k->c->arena, key); + if (!owned) { + return false; + } + cbm_ht_set(k->visited, owned, owned); + if (k->queue_len == k->queue_cap) { + int cap = k->queue_cap ? k->queue_cap * 2 : 256; + int64_t *grown = cbm_arena_alloc(&k->c->arena, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + if (k->queue_len) { + memcpy(grown, k->queue, (size_t)k->queue_len * sizeof(*grown)); + } + k->queue = grown; + k->queue_cap = cap; + } + k->queue[k->queue_len++] = id; + return true; +} + +/* *mapped: the node is a function the map knows (its id was added). */ +static bool te_cov_visit(te_cov_t *k, int64_t id, bool *mapped) { + *mapped = false; + cbm_node_t node = {0}; + if (cbm_store_find_node_by_id(k->c->store, id, &node) != CBM_STORE_OK) { + return false; + } + const cbm_coverage_function_t *f = + node.file_path && node.name + ? cbm_coverage_map_find_function(k->map, node.file_path, node.name) + : NULL; + cbm_node_free_fields(&node); + if (f) { + *mapped = true; + return te_cov_add(k, f->id); + } + return true; +} + +static bool te_cov_expand(te_cov_t *k, int64_t id) { + for (size_t t = 0; t < sizeof(te_cov_types) / sizeof(te_cov_types[0]); t++) { + cbm_edge_t *edges = NULL; + int n = 0; + if (cbm_store_find_edges_by_target_type(k->c->store, id, te_cov_types[t], &edges, &n) != + CBM_STORE_OK) { + return false; + } + bool ok = true; + for (int i = 0; ok && i < n; i++) { + ok = te_cov_push(k, edges[i].source_id); + } + cbm_store_free_edges(edges, n); + if (!ok) { + return false; + } + } + return true; +} + +static int te_cov_int_cmp(const void *a, const void *b) { + int x = *(const int *)a; + int y = *(const int *)b; + return (x > y) - (x < y); +} + +bool te_coverage_ids(te_ctx_t *c) { + c->cov_complete = false; + if (!c->coverage_admitted) { + return true; + } + te_cov_t k = {.c = c, .map = c->coverage, .visited = cbm_ht_create(1024)}; + if (!k.visited) { + return te_oom(c); + } + bool ok = true; + int seed_count = 0; + const int64_t *seeds = cbm_ti_seeds_nodes(c->seeds, &seed_count); + for (int i = 0; ok && i < seed_count; i++) { + ok = te_cov_push(&k, seeds[i]); + } + for (int head = 0; ok && head < k.queue_len; head++) { + bool mapped = false; + ok = te_cov_visit(&k, k.queue[head], &mapped) && + (mapped || te_cov_expand(&k, k.queue[head])); + } + int deleted_count = 0; + const char *const *deleted = cbm_ti_seeds_deleted_names(c->seeds, &deleted_count); + int function_count = 0; + const cbm_coverage_function_t *functions = + cbm_coverage_map_functions(c->coverage, &function_count); + for (int d = 0; ok && d < deleted_count; d++) { + for (int f = 0; ok && f < function_count; f++) { + if (strcmp(functions[f].name, deleted[d]) == 0) { + ok = te_cov_add(&k, functions[f].id); + } + } + } + cbm_ht_free(k.visited); + if (!ok) { + return true; /* incomplete: the selection then runs everything */ + } + if (k.count) { + qsort(k.ids, (size_t)k.count, sizeof(*k.ids), te_cov_int_cmp); + } + int unique = 0; + for (int i = 0; i < k.count; i++) { + if (!unique || k.ids[unique - 1] != k.ids[i]) { + k.ids[unique++] = k.ids[i]; + } + } + c->cov_ids = k.ids; + c->cov_count = unique; + c->cov_complete = true; + return true; +} + +/* ── Producer ──────────────────────────────────────────────────────── */ + +static bool te_copy_file(const char *from, const char *to) { + char *bytes = NULL; + size_t len = 0; + if (!te_read_file(from, &bytes, &len)) { + return false; + } + FILE *out = cbm_fopen(to, "wb"); + bool ok = out && fwrite(bytes, 1, len, out) == len; + if (out) { + ok = fclose(out) == 0 && ok; + } + cbm_free(CBM_MEM_CLASS_OTHER, bytes); + return ok; +} + +static bool te_publish_fail(te_ctx_t *c, const char *why) { + te_note(c, why); + return false; +} + +/* The bundle of the checked-out commit. c->rq is the request te_work_dir, + * te_git, te_config and te_snapshot read (base_ref = HEAD). */ +bool te_publish(te_ctx_t *c, const cbm_test_impact_publish_t *p) { + cbm_path_info_t info; + if (cbm_path_info_utf8(p->out_dir, &info) == 0) { + return te_publish_fail(c, "publish: the output directory already exists"); + } + if (!te_work_dir(c) || !te_git(c) || !te_config(c)) { + return te_publish_fail(c, "publish: git or configuration unavailable"); + } + cbm_ti_receipt_t previous = {0}; + if (p->previous_dir) { + const char *why = NULL; + snprintf(c->base_db, sizeof(c->base_db), "%s/graph/base.db", c->work_dir); + if (cbm_ti_receipt_read(p->previous_dir, &previous) != CBM_TI_RECEIPT_OK || + !te_bundle_import(p->previous_dir, &previous, c->base_db, &why)) { + /* An unusable previous bundle only costs time: build in full. */ + (void)cbm_unlink(c->base_db); + (void)cbm_remove_db_sidecars(c->base_db); + c->base_db[0] = '\0'; + memset(&previous, 0, sizeof(previous)); + } + } + if (!te_snapshot(c)) { + return te_publish_fail(c, "publish: the graph of HEAD could not be built"); + } + /* The store must be closed before export snapshots it. */ + cbm_ti_graph_free(c->graph); + c->graph = NULL; + cbm_store_close(c->store); + c->store = NULL; + if (!cbm_mkdir_p(p->out_dir, 0755) || + cbm_artifact_export(c->candidate_db, p->out_dir, "test-impact", CBM_ARTIFACT_BEST) != 0) { + return te_publish_fail(c, "publish: the graph could not be exported"); + } + cbm_ti_receipt_t r = {0}; + snprintf(r.commit, sizeof(r.commit), "%s", c->id->head); + snprintf(r.platform, sizeof(r.platform), "%s", p->platform ? p->platform : ""); + char check[4200]; + snprintf(check, sizeof(check), "%s/graph/check.db", c->work_dir); + if (cbm_artifact_import(p->out_dir, check) != 0 || !te_db_digest(check, r.graph_sha256) || + !te_db_content_digest(check, r.graph_content_sha256, r.graph_topology_sha256)) { + return te_publish_fail(c, "publish: the exported graph does not import"); + } + (void)cbm_unlink(check); + (void)cbm_remove_db_sidecars(check); + if (p->coverage_dir) { + char dir[4200]; + snprintf(dir, sizeof(dir), "%s/%s", p->out_dir, CBM_TI_COVERAGE_DIR); + static const char *const files[] = {"functions.tsv", "tests.tsv", "meta.json"}; + char *sums[] = {r.functions_sha256, r.tests_sha256, r.metadata_sha256}; + bool ok = cbm_mkdir_p(dir, 0755); + for (size_t i = 0; ok && i < 3; i++) { + char from[4200]; + char to[4200]; + snprintf(from, sizeof(from), "%s/%s", p->coverage_dir, files[i]); + snprintf(to, sizeof(to), "%s/%s", dir, files[i]); + ok = te_copy_file(from, to) && cbm_ti_sha256_file(to, sums[i]); + } + cbm_coverage_map_t *map = NULL; + char *meta = NULL; + size_t meta_len = 0; + ok = ok && te_load_map(dir, &map, &meta, &meta_len) && + cbm_coverage_map_metadata_matches(map, meta, meta_len, r.commit); + cbm_coverage_map_free(map); + cbm_free(CBM_MEM_CLASS_OTHER, meta); + if (!ok) { + return te_publish_fail(c, "publish: the coverage map is not this commit's"); + } + r.oldest_observation_at = p->observed_at > 0 ? p->observed_at + : previous.has_coverage ? previous.oldest_observation_at + : 0; + if (r.oldest_observation_at <= 0 || + !te_compatibility(c, p->platform, r.compatibility_sha256)) { + return te_publish_fail(c, "publish: the map's observation time or platform is unknown"); + } + r.has_coverage = true; + } + if (!cbm_ti_receipt_write(p->out_dir, &r)) { + return te_publish_fail(c, "publish: receipt.json could not be written"); + } + char text[256]; + snprintf(text, sizeof(text), "published %s route=%s topology=%s content=%s", r.commit, + c->route == CBM_PIPELINE_FROZEN_ROUTE_INCREMENTAL ? "incremental" : "full", + r.graph_topology_sha256, r.graph_content_sha256); + te_note(c, text); + return true; +} diff --git a/src/mcp/test_impact_engine_internal.h b/src/mcp/test_impact_engine_internal.h index 2fae9eaf4..604f129d7 100644 --- a/src/mcp/test_impact_engine_internal.h +++ b/src/mcp/test_impact_engine_internal.h @@ -15,6 +15,8 @@ #include "mcp/test_impact_seed.h" #include "mcp/test_impact_source.h" #include "mcp/test_impact_tree.h" +#include "mcp/test_impact_artifact.h" +#include "pipeline/pipeline_internal.h" #include "discover/test_conventions.h" #include "foundation/arena.h" #include "store/store.h" @@ -95,11 +97,33 @@ typedef struct { bool static_complete; cbm_test_selection_t *selection; + + /* Team artifact (test_impact_artifact.h). artifact_graph: a verified + * bundle of the merge base whose imported graph matched its receipt; it is + * the frozen build's incremental base (base_db). */ + bool artifact_requested; + bool artifact_graph; + cbm_ti_receipt_t receipt; + char base_db[4096]; + cbm_pipeline_frozen_route_t route; + cbm_coverage_map_t *coverage; + char *coverage_meta; + size_t coverage_meta_len; + bool coverage_admitted; + cbm_test_result_evidence_reason_t coverage_reasons[16]; + int coverage_reason_count; + int *cov_ids; /* C_eff, ascending (arena) */ + int cov_count; + bool cov_complete; } te_ctx_t; /* Record a run-all reason once; the first diagnostic wins. false only when * out of memory. */ bool te_fallback(te_ctx_t *c, cbm_test_result_fallback_t code, const char *diagnostic); +/* The first diagnostic of the request wins. */ +void te_note(te_ctx_t *c, const char *text); +bool te_git(te_ctx_t *c); +bool te_config(te_ctx_t *c); bool te_grow(void **items, int *cap, int count, size_t size); bool te_oom(te_ctx_t *c); @@ -108,6 +132,15 @@ bool te_work_dir(te_ctx_t *c); bool te_snapshot(te_ctx_t *c); void te_snapshot_free(te_ctx_t *c); +/* test_impact_engine_artifact.c */ +bool te_db_digest(const char *db_path, char hex[65]); +bool te_artifact_graph(te_ctx_t *c); +bool te_artifact_coverage(te_ctx_t *c); +bool te_coverage_ids(te_ctx_t *c); +void te_artifact_receipt(te_ctx_t *c, cbm_test_result_coverage_receipt_t *r, + cbm_test_result_oid_t (*oid)(const te_ctx_t *, const char *)); +bool te_publish(te_ctx_t *c, const cbm_test_impact_publish_t *p); + /* test_impact_engine_reach.c */ bool te_fixtures(te_ctx_t *c, int64_t **extra_seeds, int *extra_count, bool **fixture_cases); bool te_reach(te_ctx_t *c); diff --git a/src/mcp/test_impact_engine_snapshot.c b/src/mcp/test_impact_engine_snapshot.c index 0b5433774..d6b2f2024 100644 --- a/src/mcp/test_impact_engine_snapshot.c +++ b/src/mcp/test_impact_engine_snapshot.c @@ -184,15 +184,21 @@ static bool te_frozen_build(te_ctx_t *c, const char *config_bytes, size_t config .rel_path = (const char *)view->files[rows[i].file_index].path, .language = rows[i].language}; } - /* The context takes mutable strings; these are copies of the pinned facts. */ - char root[4096]; + /* The Git context is display data the build never traverses and the + * engine never reads back. It is fixed, so the candidate is a function of + * the pinned tree alone: the per-run pin path or the commit ids here land + * in the Branch node, make two builds of one tree differ, and turn every + * new commit into a semantic-input change that declines incremental repair + * of the team artifact's graph. The answer's receipt binds the commit. */ + char root[] = "/pinned"; char head[65]; char merge_base[65]; char empty[1] = ""; char branch[] = "pinned"; - snprintf(root, sizeof(root), "%s", view->root); - snprintf(head, sizeof(head), "%s", c->id->head); - snprintf(merge_base, sizeof(merge_base), "%s", c->id->merge_base); + size_t oid_len = strlen(c->id->head) == 64 ? 64 : 40; + memset(head, '0', oid_len); + head[oid_len] = '\0'; + memcpy(merge_base, head, oid_len + 1); cbm_git_context_t git = {.is_git = true, .is_detached = true, .root_exists = true, @@ -214,7 +220,8 @@ static bool te_frozen_build(te_ctx_t *c, const char *config_bytes, size_t config .config_len = config_len, .pinned_git = &git, .files = list, - .file_count = rows_n}; + .file_count = rows_n, + .base_db_path = c->base_db[0] ? c->base_db : NULL}; cbm_pipeline_frozen_t *owner = NULL; cbm_pipeline_frozen_status_t status = cbm_pipeline_frozen_create(&in, &owner); if (status == CBM_PIPELINE_FROZEN_OK) { @@ -237,6 +244,7 @@ static bool te_frozen_build(te_ctx_t *c, const char *config_bytes, size_t config error_count && errors[0].path ? errors[0].path : "", error_count ? " " : "", error_count && errors[0].reason ? errors[0].reason : ""); } + c->route = cbm_pipeline_frozen_route(owner); cbm_pipeline_frozen_free(owner); if (status != CBM_PIPELINE_FROZEN_OK) { return te_snapshot_fail(c, CBM_TEST_RESULT_FALLBACK_GRAPH_REJECTED, @@ -339,6 +347,11 @@ void te_snapshot_free(te_ctx_t *c) { snprintf(side, sizeof(side), "%s-shm", c->candidate_db); te_remove(side); } + if (c->base_db[0]) { + /* The imported team-artifact graph (test_impact_engine_artifact.c). */ + te_remove(c->base_db); + (void)cbm_remove_db_sidecars(c->base_db); + } if (c->work_dir[0]) { char sub[4200]; snprintf(sub, sizeof(sub), "%s/graph", c->work_dir); diff --git a/src/mcp/test_impact_policy.c b/src/mcp/test_impact_policy.c index a85cf5c7c..2b523908c 100644 --- a/src/mcp/test_impact_policy.c +++ b/src/mcp/test_impact_policy.c @@ -9,6 +9,8 @@ enum { TP_MAX_PATTERN = 65536, + TP_MAX_COMPAT_PATHS = 256, + TP_MAX_COMPAT_PATH = 1024, TP_MAX_EXPANSIONS = 4096, TP_MAX_EXPANDED_BYTES = 262144, TP_MAX_BRACE_DEPTH = 16, @@ -70,6 +72,8 @@ struct cbm_test_policy { const char *digest; size_t expansion_count; size_t expanded_bytes; + const char **compat_paths; + int compat_count; }; /* Expansion uses an acyclic graph with shared continuations. Its explicit @@ -719,6 +723,56 @@ static bool tp_add_builtins(cbm_test_policy_t *policy, int first) { return true; } +/* A repository-relative file path: no root, no backslash, no empty, `.` or + * `..` segment, no trailing slash. */ +static bool tp_relative_path(const char *text, size_t len) { + if (!text || !len || len > TP_MAX_COMPAT_PATH || text[0] == '/' || text[len - 1] == '/' || + memchr(text, '\\', len)) + return false; + const char *segment = text; + const char *end = text + len; + while (segment < end) { + const char *slash = memchr(segment, '/', (size_t)(end - segment)); + size_t n = (size_t)((slash ? slash : end) - segment); + if (!n || (n == 1 && segment[0] == '.') || + (n == 2 && segment[0] == '.' && segment[1] == '.')) + return false; + segment += n + 1; + } + return true; +} + +/* `coverage`: what a recorded coverage map depends on beyond the code it + * measures. Strict on keys: an ignored misspelling would make admitting a map + * easier, never harder. */ +static bool tp_parse_coverage(cbm_test_policy_t *policy, yyjson_val *coverage) { + bool valid = true; + yyjson_val *paths = tp_field(coverage, "compatibility_paths", &valid); + if (!valid || yyjson_obj_size(coverage) != (paths ? 1U : 0U) || + (paths && !yyjson_is_arr(paths))) + return false; + size_t count = paths ? yyjson_arr_size(paths) : 0; + if (count > TP_MAX_COMPAT_PATHS) + return false; + policy->compat_paths = + cbm_arena_calloc(&policy->arena, (count ? count : 1) * sizeof(*policy->compat_paths)); + if (!policy->compat_paths) + return false; + size_t index, maximum; + yyjson_val *path; + yyjson_arr_foreach(paths, index, maximum, path) { + size_t len = 0; + const char *text = tp_string(path, &len); + if (!tp_relative_path(text, len)) + return false; + policy->compat_paths[index] = cbm_arena_strdup(&policy->arena, text); + if (!policy->compat_paths[index]) + return false; + } + policy->compat_count = (int)count; + return true; +} + static bool tp_parse_policy(cbm_test_policy_t *policy, yyjson_val *impact) { bool valid = true; yyjson_val *rules = NULL, *lanes = NULL; @@ -727,8 +781,11 @@ static bool tp_parse_policy(cbm_test_policy_t *policy, yyjson_val *impact) { rules = tp_field(impact, "rules", &valid); lanes = tp_field(impact, "lanes", &valid); yyjson_val *unmapped = tp_field(impact, "unmapped_files", &valid); + yyjson_val *coverage = tp_field(impact, "coverage", &valid); if (!valid || !yyjson_is_uint(version) || yyjson_get_uint(version) != 1) return false; + if (coverage && !tp_parse_coverage(policy, coverage)) + return false; if (unmapped) { const char *text = tp_string(unmapped, NULL); if (!text || strcmp(text, "run_all") != 0) @@ -832,6 +889,13 @@ const char *cbm_test_policy_digest(const cbm_test_policy_t *policy) { return policy ? policy->digest : ""; } +const char *const *cbm_test_policy_compatibility_paths(const cbm_test_policy_t *policy, + int *count) { + if (count) + *count = policy ? policy->compat_count : 0; + return policy && policy->compat_count ? (const char *const *)policy->compat_paths : NULL; +} + typedef struct { CBMArena arena; unsigned char *first; diff --git a/src/pipeline/pass_complexity.c b/src/pipeline/pass_complexity.c index ab1574566..614694408 100644 --- a/src/pipeline/pass_complexity.c +++ b/src/pipeline/pass_complexity.c @@ -19,7 +19,11 @@ #include "foundation/log.h" #include "foundation/platform.h" #include "foundation/compat.h" +#include "foundation/arena.h" +#include "foundation/mem_core.h" +#include "store/store.h" #include "cbm.h" +#include "sqlite3.h" #include #include @@ -83,7 +87,7 @@ static void append_complexity_props(cbm_gbuf_node_t *node, int tld, bool recursi return; /* not a JSON object — leave untouched */ } bool empty = (olen == 2); /* "{}" */ - char *neu = malloc(olen + CBM_SZ_64); + char *neu = cbm_alloc(CBM_MEM_CLASS_OTHER, olen + CBM_SZ_64); if (!neu) { return; } @@ -92,11 +96,11 @@ static void append_complexity_props(cbm_gbuf_node_t *node, int tld, bool recursi snprintf(neu + (olen - 1), CBM_SZ_64, "%s\"transitive_loop_depth\":%d,\"recursive\":%s}", empty ? "" : ",", tld, recursive ? "true" : "false"); if (w < 0) { - free(neu); + cbm_free(CBM_MEM_CLASS_OTHER, neu); return; } (void)cbm_gbuf_node_set_properties_json(node, neu); - free(neu); + cbm_free(CBM_MEM_CLASS_OTHER, neu); } /* Content-only node order: qualified_name, then file path and start line. @@ -113,7 +117,13 @@ static const char *str_or_empty(const char *s) { return s ? s : ""; } -static int cmp_node_canonical(const cbm_gbuf_node_t *a, const cbm_gbuf_node_t *b) { +typedef struct { + const char *qualified_name; + const char *file_path; + int start_line; +} tld_key_t; + +static int cmp_key_canonical(const tld_key_t *a, const tld_key_t *b) { int r = strcmp(str_or_empty(a ? a->qualified_name : NULL), str_or_empty(b ? b->qualified_name : NULL)); if (r != 0) { @@ -131,18 +141,80 @@ static int cmp_node_canonical(const cbm_gbuf_node_t *a, const cbm_gbuf_node_t *b return 0; } -static int cmp_seed_canonical(const void *pa, const void *pb) { - return cmp_node_canonical(*(const cbm_gbuf_node_t *const *)pa, - *(const cbm_gbuf_node_t *const *)pb); -} - typedef struct { - const cbm_gbuf_node_t *node; /* NULL for a dangling target id */ + const tld_key_t *key; /* NULL for a dangling target id */ int64_t id; } tld_callee_t; static int cmp_callee_canonical(const void *pa, const void *pb) { - return cmp_node_canonical(((const tld_callee_t *)pa)->node, ((const tld_callee_t *)pb)->node); + return cmp_key_canonical(((const tld_callee_t *)pa)->key, ((const tld_callee_t *)pb)->key); +} + +/* The call graph the traversal reads, whoever gathered it: the graph buffer + * (a full build) or the staging store (the closure-delta incremental route). + * One traversal for both is what keeps an incremental graph equal to a full + * one. Ids index every array directly; first[] is a CSR of CALLS edges by + * source id in gathering order (each frame sorts its callees canonically). */ +typedef struct { + int64_t maxid; + tld_key_t *keys; /* [maxid + 1] */ + bool *present; /* [maxid + 1]: a node exists for the id */ + int *loop_depth; /* [maxid + 1]: seeds only */ + bool *recursive; /* [maxid + 1]: self_recursive, then cycles */ + tld_callee_t *seeds; /* Function/Method nodes */ + int seed_count; + int *first; /* [maxid + 2] */ + int64_t *callee; /* [edge_count] */ + int edge_count; +} tld_graph_t; + +static void tld_graph_free(tld_graph_t *g) { + cbm_free(CBM_MEM_CLASS_OTHER, g->keys); + cbm_free(CBM_MEM_CLASS_OTHER, g->present); + cbm_free(CBM_MEM_CLASS_OTHER, g->loop_depth); + cbm_free(CBM_MEM_CLASS_OTHER, g->recursive); + cbm_free(CBM_MEM_CLASS_OTHER, g->seeds); + cbm_free(CBM_MEM_CLASS_OTHER, g->first); + cbm_free(CBM_MEM_CLASS_OTHER, g->callee); +} + +static bool tld_graph_alloc(tld_graph_t *g, int64_t maxid, int seed_cap, int edge_count) { + size_t sz = (size_t)maxid + 1; + g->maxid = maxid; + g->keys = cbm_calloc(CBM_MEM_CLASS_OTHER, sz * sizeof(*g->keys)); + g->present = cbm_calloc(CBM_MEM_CLASS_OTHER, sz * sizeof(*g->present)); + g->loop_depth = cbm_calloc(CBM_MEM_CLASS_OTHER, sz * sizeof(*g->loop_depth)); + g->recursive = cbm_calloc(CBM_MEM_CLASS_OTHER, sz * sizeof(*g->recursive)); + g->seeds = cbm_calloc(CBM_MEM_CLASS_OTHER, ((size_t)seed_cap + 1) * sizeof(*g->seeds)); + g->first = cbm_calloc(CBM_MEM_CLASS_OTHER, (sz + 1) * sizeof(*g->first)); + g->callee = cbm_calloc(CBM_MEM_CLASS_OTHER, ((size_t)edge_count + 1) * sizeof(*g->callee)); + return g->keys && g->present && g->loop_depth && g->recursive && g->seeds && g->first && + g->callee; +} + +/* Sources must be counted into first[src + 1] before this; turns the counts + * into offsets. */ +static void tld_graph_offsets(tld_graph_t *g) { + for (int64_t id = 1; id <= g->maxid + 1; id++) { + g->first[id] += g->first[id - 1]; + } +} + +static void tld_key_set(tld_graph_t *g, int64_t id, const char *qn, const char *file, int line) { + if (id >= 1 && id <= g->maxid) { + g->keys[id] = (tld_key_t){qn, file, line}; + g->present[id] = true; + } +} + +static void tld_seed(tld_graph_t *g, int64_t id, const char *properties, int seed_cap) { + if (id >= 1 && id <= g->maxid && g->seed_count < seed_cap) { + g->loop_depth[id] = json_get_int(properties, "loop_depth", 0); + g->recursive[id] = json_get_bool(properties, "self_recursive"); + g->seeds[g->seed_count].id = id; + g->seeds[g->seed_count].key = &g->keys[id]; + g->seed_count++; + } } /* Traversal state. `callees` is one bump stack shared by every DFS frame: a @@ -151,62 +223,46 @@ static int cmp_callee_canonical(const void *pa, const void *pb) { * re-entry), so the live slots never exceed the CALLS edge count the stack is * sized for. */ typedef struct { - const cbm_gbuf_t *gb; - int64_t maxid; - int *loop_depth; + tld_graph_t *g; int *tld; char *state; - bool *recursive; - cbm_gbuf_node_t **seeds; /* Function/Method nodes, canonical order */ - int seed_count; - int seed_cap; tld_callee_t *callees; int callee_top; int callee_cap; } tld_ctx_t; -static void tld_ctx_free(tld_ctx_t *cx) { - free(cx->loop_depth); - free(cx->tld); - free(cx->state); - free(cx->recursive); - free(cx->seeds); - free(cx->callees); -} - /* Memoized DFS: tld(id) = loop_depth(id) + max over CALLS-callees of tld(callee). * state: 0=unvisited, 1=in-progress (back-edge -> cycle), 2=done. */ static int tld_dfs(tld_ctx_t *cx, int64_t id, int depth) { - if (id < 1 || id > cx->maxid) { + tld_graph_t *g = cx->g; + if (id < 1 || id > g->maxid) { return 0; } if (cx->state[id] == 2) { return cx->tld[id]; } if (cx->state[id] == 1) { - cx->recursive[id] = true; /* back edge -> call-graph cycle */ + g->recursive[id] = true; /* back edge -> call-graph cycle */ return 0; } if (depth > CBM_TLD_MAX_DEPTH) { - return cx->loop_depth[id]; + return g->loop_depth[id]; } - const cbm_gbuf_edge_t **edges = NULL; - int ne = 0; - cbm_gbuf_find_edges_by_source_type(cx->gb, id, "CALLS", &edges, &ne); + int ne = g->first[id + 1] - g->first[id]; if (ne > cx->callee_cap - cx->callee_top) { - return cx->loop_depth[id]; /* unreachable by construction; same as the depth cap */ + return g->loop_depth[id]; /* unreachable by construction; same as the depth cap */ } cx->state[id] = 1; tld_callee_t *callees = cx->callees + cx->callee_top; int nc = 0; - for (int i = 0; i < ne; i++) { - int64_t c = edges[i]->target_id; + for (int i = g->first[id]; i < g->first[id + 1]; i++) { + int64_t c = g->callee[i]; if (c == id) { - cx->recursive[id] = true; /* direct self-recursion */ + g->recursive[id] = true; /* direct self-recursion */ continue; } callees[nc].id = c; - callees[nc].node = cbm_gbuf_find_by_id(cx->gb, c); + callees[nc].key = c >= 1 && c <= g->maxid && g->present[c] ? &g->keys[c] : NULL; nc++; } cx->callee_top += nc; @@ -219,11 +275,37 @@ static int tld_dfs(tld_ctx_t *cx, int64_t id, int depth) { } } cx->callee_top -= nc; - cx->tld[id] = cx->loop_depth[id] + best; + cx->tld[id] = g->loop_depth[id] + best; cx->state[id] = 2; return cx->tld[id]; } +/* Runs the traversal from every seed in canonical order; *tld_out[id] and + * g->recursive[id] then hold each seed's result. */ +static bool tld_run(tld_graph_t *g, int **tld_out) { + size_t sz = (size_t)g->maxid + 1; + tld_ctx_t cx = {.g = g, .callee_cap = g->edge_count}; + cx.tld = cbm_calloc(CBM_MEM_CLASS_OTHER, sz * sizeof(*cx.tld)); + cx.state = cbm_calloc(CBM_MEM_CLASS_OTHER, sz * sizeof(*cx.state)); + cx.callees = cbm_calloc(CBM_MEM_CLASS_OTHER, ((size_t)g->edge_count + 1) * sizeof(*cx.callees)); + if (!cx.tld || !cx.state || !cx.callees) { + cbm_free(CBM_MEM_CLASS_OTHER, cx.tld); + cbm_free(CBM_MEM_CLASS_OTHER, cx.state); + cbm_free(CBM_MEM_CLASS_OTHER, cx.callees); + return false; + } + qsort(g->seeds, (size_t)g->seed_count, sizeof(*g->seeds), cmp_callee_canonical); + for (int i = 0; i < g->seed_count; i++) { + if (cx.state[g->seeds[i].id] != 2) { + tld_dfs(&cx, g->seeds[i].id, 0); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, cx.state); + cbm_free(CBM_MEM_CLASS_OTHER, cx.callees); + *tld_out = cx.tld; + return true; +} + static int label_count(const cbm_gbuf_t *gb, const char *label) { const cbm_gbuf_node_t **nodes = NULL; int count = 0; @@ -233,75 +315,320 @@ static int label_count(const cbm_gbuf_t *gb, const char *label) { return count; } -static int calls_edge_count(const cbm_gbuf_t *gb) { - const cbm_gbuf_edge_t **edges = NULL; - int count = 0; - if (cbm_gbuf_find_edges_by_type(gb, "CALLS", &edges, &count) != 0) { - return 0; - } - return count; -} - -/* Seed each Function/Method node's loop_depth and self_recursive flag, and - * collect the node as a traversal seed (also the write-back target). The - * self_recursive seed (set at extraction) feeds the final recursive flag; - * tld_dfs additionally ORs in mutual recursion discovered as a call-graph - * cycle. */ -static void seed_loop_depths(tld_ctx_t *cx, const char *label) { - const cbm_gbuf_node_t **nodes = NULL; - int count = 0; - if (cbm_gbuf_find_by_label(cx->gb, label, &nodes, &count) != 0) { - return; - } - for (int i = 0; i < count && cx->seed_count < cx->seed_cap; i++) { - const cbm_gbuf_node_t *n = nodes[i]; - if (n->id >= 1 && n->id <= cx->maxid) { - cx->loop_depth[n->id] = json_get_int(n->properties_json, "loop_depth", 0); - cx->recursive[n->id] = json_get_bool(n->properties_json, "self_recursive"); - cx->seeds[cx->seed_count++] = (cbm_gbuf_node_t *)n; - } - } -} - -void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx) { - cbm_gbuf_t *gb = ctx->gbuf; +/* Gathers the graph buffer: every Function/Method node as a seed, every CALLS + * edge, and the key of every node a CALLS edge targets. */ +static bool tld_gather_gbuf(const cbm_gbuf_t *gb, tld_graph_t *g) { /* Node and edge IDs are drawn from one shared counter, so node IDs are NOT * contiguous 1..node_count — they interleave with edge IDs. Size the lookup * arrays by the id ceiling (next_id) so every node id is addressable. */ int64_t maxid = cbm_gbuf_next_id(gb) - 1; - if (maxid < 1) { - return; + const cbm_gbuf_edge_t **edges = NULL; + int edge_count = 0; + if (maxid < 1 || cbm_gbuf_find_edges_by_type(gb, "CALLS", &edges, &edge_count) != 0) { + edge_count = 0; } - size_t sz = (size_t)maxid + 1; - tld_ctx_t cx = {0}; - cx.gb = gb; - cx.maxid = maxid; - cx.seed_cap = label_count(gb, "Function") + label_count(gb, "Method"); - cx.callee_cap = calls_edge_count(gb); - cx.loop_depth = calloc(sz, sizeof(int)); - cx.tld = calloc(sz, sizeof(int)); - cx.state = calloc(sz, sizeof(char)); - cx.recursive = calloc(sz, sizeof(bool)); - cx.seeds = calloc((size_t)cx.seed_cap + 1, sizeof(cbm_gbuf_node_t *)); - cx.callees = calloc((size_t)cx.callee_cap + 1, sizeof(tld_callee_t)); - if (!cx.loop_depth || !cx.tld || !cx.state || !cx.recursive || !cx.seeds || !cx.callees) { - tld_ctx_free(&cx); - return; + int seed_cap = label_count(gb, "Function") + label_count(gb, "Method"); + if (maxid < 1 || !tld_graph_alloc(g, maxid, seed_cap, edge_count)) { + return false; } - - seed_loop_depths(&cx, "Function"); - seed_loop_depths(&cx, "Method"); - qsort(cx.seeds, (size_t)cx.seed_count, sizeof(*cx.seeds), cmp_seed_canonical); - - for (int i = 0; i < cx.seed_count; i++) { - cbm_gbuf_node_t *n = cx.seeds[i]; - if (cx.state[n->id] != 2) { - tld_dfs(&cx, n->id, 0); + static const char *const labels[] = {"Function", "Method"}; + for (size_t l = 0; l < sizeof(labels) / sizeof(labels[0]); l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(gb, labels[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + tld_key_set(g, n->id, n->qualified_name, n->file_path, n->start_line); + tld_seed(g, n->id, n->properties_json, seed_cap); } - append_complexity_props(n, cx.tld[n->id], cx.recursive[n->id]); } + for (int i = 0; i < edge_count; i++) { + int64_t source = edges[i]->source_id; + int64_t target = edges[i]->target_id; + if (source >= 1 && source <= maxid) { + g->first[source + 1]++; + } + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gb, target); + if (t && !(target >= 1 && target <= maxid && g->present[target])) { + tld_key_set(g, target, t->qualified_name, t->file_path, t->start_line); + } + } + tld_graph_offsets(g); + int *fill = cbm_calloc(CBM_MEM_CLASS_OTHER, ((size_t)maxid + 1) * sizeof(*fill)); + if (!fill) { + return false; + } + for (int i = 0; i < edge_count; i++) { + int64_t source = edges[i]->source_id; + if (source >= 1 && source <= maxid) { + g->callee[g->first[source] + fill[source]++] = edges[i]->target_id; + } + } + cbm_free(CBM_MEM_CLASS_OTHER, fill); + g->edge_count = edge_count; + return true; +} - cbm_log_info("pass.complexity", "functions", itoa_cx(cx.seed_count)); +void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx) { + cbm_gbuf_t *gb = ctx->gbuf; + tld_graph_t g = {0}; + int *tld = NULL; + if (!tld_gather_gbuf(gb, &g) || !tld_run(&g, &tld)) { + tld_graph_free(&g); + return; + } + for (int i = 0; i < g.seed_count; i++) { + int64_t id = g.seeds[i].id; + cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)cbm_gbuf_find_by_id(gb, id); + if (n) { + append_complexity_props(n, tld[id], g.recursive[id]); + } + } + cbm_log_info("pass.complexity", "functions", itoa_cx(g.seed_count)); + cbm_free(CBM_MEM_CLASS_OTHER, tld); + tld_graph_free(&g); +} - tld_ctx_free(&cx); +/* ── Store-level recompute (closure-delta incremental) ──────────────── + * + * The closure-delta route patches the staging store with the re-extracted + * files only; transitive_loop_depth and recursive are properties of the whole + * call graph, so a body edit can move them for callers in files that were not + * re-parsed, and the re-parsed nodes arrive without them. Like importance, the + * recompute therefore runs over the complete staging store after the patch, + * with the same traversal as the in-memory pass (only the gathering differs) + * and BEFORE the importance rescore, so the keys land where a full build puts + * them. Existing keys are rewritten in place; absent ones are appended. A + * failure fails the delta route, which then rebuilds in full. */ + +/* properties with transitive_loop_depth/recursive set: rewritten in place + * when present, appended exactly as the in-memory pass appends otherwise. + * NULL: allocation failure or a malformed existing pair. */ +static char *complexity_set_props(const char *old, int tld, bool recursive) { + static const char key[] = "\"transitive_loop_depth\":"; + static const char rec_key[] = ",\"recursive\":"; + const char *props = old ? old : "{}"; + size_t olen = strlen(props); + char value[CBM_SZ_64]; + int w = + snprintf(value, sizeof(value), "%s%d%s%s", key, tld, rec_key, recursive ? "true" : "false"); + if (w < 0 || (size_t)w >= sizeof(value)) { + return NULL; + } + const char *at = strstr(props, key); + size_t head = 0; + size_t tail = 0; /* offset where the old pair ends */ + const char *sep = ""; + if (at) { + const char *p = at + sizeof(key) - 1; + if (*p == '-') { + p++; + } + while (*p >= '0' && *p <= '9') { + p++; + } + if (strncmp(p, rec_key, sizeof(rec_key) - 1) != 0) { + return NULL; + } + p += sizeof(rec_key) - 1; + if (strncmp(p, "true", 4) == 0) { + p += 4; + } else if (strncmp(p, "false", 5) == 0) { + p += 5; + } else { + return NULL; + } + head = (size_t)(at - props); + tail = (size_t)(p - props); + } else { + if (olen < 2 || props[olen - 1] != '}') { + return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, props); /* not an object: untouched */ + } + head = olen - 1; + tail = olen - 1; + sep = olen == 2 ? "" : ","; + } + size_t len = head + strlen(sep) + (size_t)w + (olen - tail); + char *neu = cbm_alloc(CBM_MEM_CLASS_OTHER, len + 1); + if (!neu) { + return NULL; + } + memcpy(neu, props, head); + size_t off = head; + memcpy(neu + off, sep, strlen(sep)); + off += strlen(sep); + memcpy(neu + off, value, (size_t)w); + off += (size_t)w; + memcpy(neu + off, props + tail, olen - tail); + neu[len] = '\0'; + return neu; +} + +/* UDF: cbm_cx_set(properties, tld, recursive). */ +static void sql_complexity_set(sqlite3_context *c, int argc, sqlite3_value **argv) { + if (argc != 3) { + sqlite3_result_error(c, "cbm_cx_set arity", -1); + return; + } + char *neu = complexity_set_props((const char *)sqlite3_value_text(argv[0]), + sqlite3_value_int(argv[1]), sqlite3_value_int(argv[2]) != 0); + if (!neu) { + sqlite3_result_error(c, "cbm_cx_set: malformed complexity properties", -1); + return; + } + sqlite3_result_text(c, neu, -1, SQLITE_TRANSIENT); + cbm_free(CBM_MEM_CLASS_OTHER, neu); +} + +static bool tld_step_int64(sqlite3_stmt *st, int64_t *out) { + if (sqlite3_step(st) != SQLITE_ROW) { + return false; + } + *out = sqlite3_column_int64(st, 0); + return true; +} + +/* Gathers the staging store the way tld_gather_gbuf gathers the buffer. Key + * strings are copied into `arena`. */ +static bool tld_gather_store(sqlite3 *db, const char *project, CBMArena *arena, tld_graph_t *g) { + static const char kMax[] = "SELECT COALESCE(MAX(id), 0) FROM nodes WHERE project = ?1"; + static const char kCounts[] = + "SELECT (SELECT COUNT(*) FROM nodes WHERE project = ?1 AND label IN " + "('Function','Method')), (SELECT COUNT(*) FROM edges WHERE project = ?1 AND type = " + "'CALLS')"; + static const char kNodes[] = + "SELECT id, label, qualified_name, file_path, start_line, properties FROM nodes " + "WHERE project = ?1 AND (label IN ('Function','Method') OR id IN (SELECT target_id " + "FROM edges WHERE project = ?1 AND type = 'CALLS'))"; + static const char kEdges[] = + "SELECT source_id, target_id FROM edges WHERE project = ?1 AND type = 'CALLS' ORDER BY id"; + sqlite3_stmt *st = NULL; + int64_t maxid = 0; + bool ok = sqlite3_prepare_v2(db, kMax, -1, &st, NULL) == SQLITE_OK && + sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT) == SQLITE_OK && + tld_step_int64(st, &maxid); + sqlite3_finalize(st); + st = NULL; + int seed_cap = 0; + int edge_count = 0; + ok = ok && sqlite3_prepare_v2(db, kCounts, -1, &st, NULL) == SQLITE_OK && + sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT) == SQLITE_OK && + sqlite3_step(st) == SQLITE_ROW; + if (ok) { + seed_cap = sqlite3_column_int(st, 0); + edge_count = sqlite3_column_int(st, 1); + } + sqlite3_finalize(st); + st = NULL; + if (!ok || maxid < 1) { + return ok; /* an empty project: nothing to recompute */ + } + if (!tld_graph_alloc(g, maxid, seed_cap, edge_count)) { + return false; + } + ok = sqlite3_prepare_v2(db, kNodes, -1, &st, NULL) == SQLITE_OK && + sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT) == SQLITE_OK; + int rc = SQLITE_ROW; + while (ok && (rc = sqlite3_step(st)) == SQLITE_ROW) { + int64_t id = sqlite3_column_int64(st, 0); + const char *label = (const char *)sqlite3_column_text(st, 1); + const char *qn = (const char *)sqlite3_column_text(st, 2); + const char *file = (const char *)sqlite3_column_text(st, 3); + char *qn_copy = qn ? cbm_arena_strdup(arena, qn) : NULL; + char *file_copy = file ? cbm_arena_strdup(arena, file) : NULL; + ok = (!qn || qn_copy) && (!file || file_copy) && id >= 1 && id <= maxid; + if (!ok) { + break; + } + tld_key_set(g, id, qn_copy, file_copy, sqlite3_column_int(st, 4)); + if (label && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0)) { + tld_seed(g, id, (const char *)sqlite3_column_text(st, 5), seed_cap); + } + } + ok = ok && rc == SQLITE_DONE; + sqlite3_finalize(st); + st = NULL; + int64_t *sources = + ok ? cbm_alloc(CBM_MEM_CLASS_OTHER, ((size_t)edge_count + 1) * sizeof(*sources)) : NULL; + ok = ok && sources && sqlite3_prepare_v2(db, kEdges, -1, &st, NULL) == SQLITE_OK && + sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT) == SQLITE_OK; + int n = 0; + while (ok && (rc = sqlite3_step(st)) == SQLITE_ROW) { + int64_t source = sqlite3_column_int64(st, 0); + int64_t target = sqlite3_column_int64(st, 1); + ok = n < edge_count && source >= 1 && source <= maxid; + if (ok) { + sources[n] = source; + g->callee[n] = target; + g->first[source + 1]++; + n++; + } + } + ok = ok && rc == SQLITE_DONE && n == edge_count; + sqlite3_finalize(st); + if (ok) { + /* Rows arrived in edge order; place each one by its source. */ + tld_graph_offsets(g); + int64_t *ordered = cbm_alloc(CBM_MEM_CLASS_OTHER, ((size_t)n + 1) * sizeof(*ordered)); + int *fill = cbm_calloc(CBM_MEM_CLASS_OTHER, ((size_t)maxid + 1) * sizeof(*fill)); + ok = ordered && fill; + for (int i = 0; ok && i < n; i++) { + ordered[g->first[sources[i]] + fill[sources[i]]++] = g->callee[i]; + } + if (ok) { + memcpy(g->callee, ordered, (size_t)n * sizeof(*ordered)); + } + cbm_free(CBM_MEM_CLASS_OTHER, ordered); + cbm_free(CBM_MEM_CLASS_OTHER, fill); + } + cbm_free(CBM_MEM_CLASS_OTHER, sources); + g->edge_count = n; + return ok; +} + +int cbm_pipeline_complexity_recompute_store(cbm_store_t *store, const char *project) { + sqlite3 *db = store && project ? cbm_store_get_db(store) : NULL; + if (!db) { + return -1; + } + if (sqlite3_create_function(db, "cbm_cx_set", 3, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, + sql_complexity_set, NULL, NULL) != SQLITE_OK) { + cbm_log_error("complexity.store_udf_failed", "err", sqlite3_errmsg(db)); + return -1; + } + CBMArena arena; + cbm_arena_init_lazy(&arena, CBM_ARENA_DEFAULT_BLOCK_SIZE); + tld_graph_t g = {0}; + int *tld = NULL; + bool ok = tld_gather_store(db, project, &arena, &g) && (!g.maxid || tld_run(&g, &tld)); + if (ok && g.seed_count && cbm_store_begin(store) == CBM_STORE_OK) { + sqlite3_stmt *st = NULL; + ok = sqlite3_prepare_v2(db, + "UPDATE nodes SET properties = cbm_cx_set(properties, ?1, ?2) " + "WHERE id = ?3", + -1, &st, NULL) == SQLITE_OK; + for (int i = 0; ok && i < g.seed_count; i++) { + int64_t id = g.seeds[i].id; + ok = sqlite3_bind_int(st, 1, tld[id]) == SQLITE_OK && + sqlite3_bind_int(st, 2, g.recursive[id] ? 1 : 0) == SQLITE_OK && + sqlite3_bind_int64(st, 3, id) == SQLITE_OK && sqlite3_step(st) == SQLITE_DONE && + sqlite3_reset(st) == SQLITE_OK; + } + sqlite3_finalize(st); + if (!ok) { + cbm_log_error("complexity.store_update_failed", "err", sqlite3_errmsg(db)); + (void)cbm_store_rollback(store); + } else { + ok = cbm_store_commit(store) == CBM_STORE_OK; + } + } else if (ok && g.seed_count) { + ok = false; + } + cbm_log_info("pass.complexity_store", "functions", itoa_cx(g.seed_count)); + cbm_free(CBM_MEM_CLASS_OTHER, tld); + tld_graph_free(&g); + cbm_arena_destroy(&arena); + return ok ? 0 : -1; } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 5aebed6c0..139c68cf3 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -291,6 +291,9 @@ struct cbm_pipeline_frozen { bool has_list; cbm_pipeline_frozen_file_t *listed; size_t listed_count; + /* Arena copy of inputs->base_db_path; NULL builds from scratch. */ + const char *base_db_path; + cbm_pipeline_frozen_route_t route; char canonical_root[CBM_SZ_4K]; char canonical_parent[CBM_SZ_4K]; #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS @@ -449,7 +452,8 @@ static bool frozen_inputs_valid(const cbm_pipeline_frozen_inputs_t *in) { return in && frozen_absolute(in->source_root) && frozen_absolute(in->candidate_db_path) && frozen_string(in->project, true) && cbm_validate_project_name(in->project) && frozen_git_valid(in->pinned_git) && frozen_policy_valid(&in->resource_policy) && - frozen_config_span_valid(in) && frozen_files_valid(in); + frozen_config_span_valid(in) && frozen_files_valid(in) && + (!in->base_db_path || frozen_absolute(in->base_db_path)); } static int frozen_listed_cmp(const void *left, const void *right) { @@ -656,6 +660,12 @@ cbm_pipeline_frozen_status_t cbm_pipeline_frozen_create(const cbm_pipeline_froze owner->arena = arena; owner->external_cancelled = inputs->cancelled; cbm_pipeline_frozen_status_t status = frozen_copy_list(owner, inputs); + if (status == CBM_PIPELINE_FROZEN_OK && inputs->base_db_path) { + owner->base_db_path = cbm_arena_strdup(&owner->arena, inputs->base_db_path); + if (!owner->base_db_path) { + status = CBM_PIPELINE_FROZEN_OOM; + } + } if (status == CBM_PIPELINE_FROZEN_OK) { status = frozen_allocate_pipeline(owner, inputs); } @@ -676,6 +686,11 @@ const cbm_pipeline_t *cbm_pipeline_frozen_diagnostics(const cbm_pipeline_frozen_ return owner ? owner->pipeline : NULL; } +cbm_pipeline_frozen_route_t cbm_pipeline_frozen_route(const cbm_pipeline_frozen_t *owner) { + return owner && owner->state == FROZEN_CANDIDATE ? owner->route + : CBM_PIPELINE_FROZEN_ROUTE_FULL; +} + bool cbm_pipeline_frozen_negative_evidence(const cbm_pipeline_frozen_t *owner, unsigned *out_flags) { if (out_flags) @@ -3521,9 +3536,13 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { } /* Check for existing DB → try incremental or delete for reindex */ - rc = p->frozen ? CBM_PIPELINE_FORCE_FULL_REINDEX - : try_incremental_or_delete_db(p, files, file_count, baseline_manifest, - baseline_count, mode_promoted); + rc = p->frozen && !p->frozen->base_db_path + ? CBM_PIPELINE_FORCE_FULL_REINDEX + : try_incremental_or_delete_db(p, files, file_count, baseline_manifest, baseline_count, + mode_promoted); + if (p->frozen && rc >= 0) { + p->frozen->route = CBM_PIPELINE_FROZEN_ROUTE_INCREMENTAL; + } if (rc == CBM_PIPELINE_ABORT_PRESERVE_DB || rc == CBM_PIPELINE_PERSIST_FAILED) { goto cleanup; } @@ -4417,8 +4436,16 @@ static cbm_pipeline_frozen_status_t frozen_run_stage(cbm_pipeline_t *p, const ch if (!stage_copy) return CBM_PIPELINE_FROZEN_OOM; p->db_path = stage_copy; - p->existing_generation = false; - p->final_existed = false; + /* A base candidate becomes the stage's existing generation, so the route + * below may repair it instead of rebuilding. */ + bool based = p->frozen->base_db_path != NULL; + if (based && cbm_store_backup_path(p->frozen->base_db_path, stage_copy) != CBM_STORE_OK) { + p->db_path = destination; + cbm_free(CBM_MEM_CLASS_OTHER, stage_copy); + return CBM_PIPELINE_FROZEN_IO; + } + p->existing_generation = based; + p->final_existed = based; int rc = cbm_pipeline_run_staged(p); p->db_path = destination; cbm_free(CBM_MEM_CLASS_OTHER, stage_copy); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 3aae16c25..f2d5d603f 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1412,7 +1412,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed * a full build takes, which is what makes its output converge. */ #define MIN_FILES_FOR_PARALLEL_INCR 50 - int worker_count = cbm_default_worker_count(true); + int worker_count = cbm_pipeline_is_frozen(ctx->pipeline) ? 1 : cbm_default_worker_count(true); bool use_parallel = closure != NULL || (worker_count > SKIP_ONE && ci > MIN_FILES_FOR_PARALLEL_INCR); @@ -2515,6 +2515,14 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char * FORCE_FULL_REINDEX, so the run degrades to a correct full rebuild * rather than publishing wrong scores. */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); + if (cbm_pipeline_complexity_recompute_store(staging, project) != 0) { + cbm_log_error("delta.err", "phase", "complexity_recompute"); + goto out; + } + cbm_log_info("pass.timing", "pass", "delta_complexity_store", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + + cbm_clock_gettime(CLOCK_MONOTONIC, &t); if (cbm_pipeline_importance_recompute_store(staging, project) != 0) { cbm_log_error("delta.err", "phase", "importance_recompute"); goto out; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7186fd345..e0c37f781 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -90,8 +90,22 @@ typedef struct { * there is INPUT_CHANGED. NULL with 0 discovers source_root. Copied. */ const cbm_pipeline_frozen_file_t *files; size_t file_count; + + /* Optional: an earlier candidate of the SAME project label (the team + * artifact's graph, imported). The build copies it into its stage and routes + * through the incremental closure repair; every doubt there declines to the + * full build, so the candidate is the full build's graph either way. + * Absolute, a regular SQLite file outside source_root; never modified. + * NULL builds from scratch. Copied. */ + const char *base_db_path; } cbm_pipeline_frozen_inputs_t; +/* Which route built the candidate. A base whose repair declined is FULL. */ +typedef enum { + CBM_PIPELINE_FROZEN_ROUTE_FULL = 0, + CBM_PIPELINE_FROZEN_ROUTE_INCREMENTAL = 1, +} cbm_pipeline_frozen_route_t; + /* Clear *out first. Deep-copy/validate all input; no Git/config filesystem * reads, no pipeline/global changes, no directory/DB creation. Caller input * buffers may be destroyed after return. NULL result on any failure. @@ -128,6 +142,8 @@ void cbm_pipeline_frozen_cancel(cbm_pipeline_frozen_t *owner); * pass this to a mutating ordinary-pipeline API. No completeness getter exists. */ const cbm_pipeline_t *cbm_pipeline_frozen_diagnostics(const cbm_pipeline_frozen_t *owner); +/* FULL until a build produced a candidate; then the route that built it. */ +cbm_pipeline_frozen_route_t cbm_pipeline_frozen_route(const cbm_pipeline_frozen_t *owner); /* Monotone negative observations; zero is NOT a completeness certificate. * PARSE_GAP alone does not refuse the candidate: a file that parsed only @@ -1070,6 +1086,11 @@ void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score); * a delta-route failure and falls back to a full rebuild. */ int cbm_pipeline_importance_recompute_store(cbm_store_t *store, const char *project); +/* transitive_loop_depth/recursive over the complete staging store of the + * closure-delta route (the in-memory pass's traversal, gathered from SQL). + * Runs before the importance rescore. 0 on success. */ +int cbm_pipeline_complexity_recompute_store(cbm_store_t *store, const char *project); + /* Work counters for the importance pass (pass_importance.c). Deltas are read * by the complexity suite's linearity gate; never reset by the pass itself, so * nested/repeated runs compose. g_importance_name_visits counts same-name-group diff --git a/src/store/store.h b/src/store/store.h index 6a90c0073..291faabdd 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -414,6 +414,16 @@ void cbm_store_close(cbm_store_t *s); /* Get the underlying sqlite3 handle (for testing only). */ struct sqlite3 *cbm_store_get_db(cbm_store_t *s); +/* The digest of a project's graph CONTENT (store_content_digest.c): rows keyed + * by what they say, never by row id, database identity, generation or mtime. + * Two graphs of the same tree, built incrementally or in full, are equal + * exactly when their content digests are. out: 32 raw SHA-256 bytes. */ +int cbm_store_graph_content_digest(cbm_store_t *s, const char *project, unsigned char out[32]); +/* The part of the content test selection reads: node identity, every edge by + * endpoints and type without properties, except the corpus-statistical + * SIMILAR_TO / SEMANTICALLY_RELATED; file hashes; index coverage. */ +int cbm_store_graph_topology_digest(cbm_store_t *s, const char *project, unsigned char out[32]); + /* Get the last error message (static string, valid until next call). */ const char *cbm_store_error(cbm_store_t *s); diff --git a/src/store/store_content_digest.c b/src/store/store_content_digest.c new file mode 100644 index 000000000..c9287adee --- /dev/null +++ b/src/store/store_content_digest.c @@ -0,0 +1,178 @@ +/* + * store_content_digest.c — the digest of a project's graph CONTENT. + * + * cbm_store_graph_digest binds one database exactly: row ids, the random + * db_uid, generation counters and file mtimes included. That is right for + * proving an imported copy is the published one, and useless for asking + * whether two independently built graphs are the same graph. This digest + * answers the second question: every row is keyed by what it says, never by + * where it was stored. Nodes by their own columns; edges by the content keys + * of their endpoints; file hashes by path, content hash and size; LSP + * surfaces, coverage rows and summaries without their timestamps. An + * incremental index equals a full one exactly when their content digests are + * equal. + */ +#include "store/store.h" + +#include "foundation/sha256.h" +#include "sqlite3.h" + +#include +#include +#include +#include + +typedef struct { + const char *name; + const char *sql; /* ?1 = project; every column is hashed, rows in this order */ +} sc_query_t; + +static const sc_query_t sc_queries[] = { + {"nodes", "SELECT label,name,qualified_name,file_path,start_line,end_line,properties " + "FROM main.nodes WHERE project=?1 ORDER BY 3,1,4,5,6,2,7;"}, + {"edges", "SELECT s.label,s.qualified_name,s.file_path,s.start_line,e.type,t.label," + "t.qualified_name,t.file_path,t.start_line,e.properties,e.url_path_gen," + "e.local_name_gen FROM main.edges e JOIN main.nodes s ON s.id=e.source_id " + "JOIN main.nodes t ON t.id=e.target_id WHERE e.project=?1 " + "ORDER BY 2,1,3,4,5,7,6,8,9,10,11,12;"}, + {"dangling_edges", "SELECT COUNT(*) FROM main.edges e WHERE e.project=?1 AND (NOT EXISTS " + "(SELECT 1 FROM main.nodes n WHERE n.id=e.source_id) OR NOT EXISTS " + "(SELECT 1 FROM main.nodes n WHERE n.id=e.target_id));"}, + {"file_hashes", "SELECT rel_path,sha256,size FROM main.file_hashes WHERE project=?1 " + "ORDER BY 1,2,3;"}, + {"lsp_surface", "SELECT rel_path,surface_sha,defs_json,ref_bloom,config_ctx FROM " + "main.lsp_surface WHERE project=?1 ORDER BY 1,2,3,4,5;"}, + {"index_coverage", "SELECT rel_path,kind,detail FROM main.index_coverage WHERE project=?1 " + "ORDER BY 1,2,3;"}, + {"index_coverage_meta", "SELECT index_mode,recording_status,ignored_files_stored," + "ignored_files_total,coverage_version,hash_records_complete FROM " + "main.index_coverage_meta WHERE project=?1;"}, + {"project_summaries", "SELECT summary,source_hash FROM main.project_summaries WHERE " + "project=?1;"}, +}; + +/* What test selection reads: node identity and every edge by its endpoints + * and type, but no edge properties, and not the two corpus-statistical edge + * types (similarity over the whole corpus, which the closure-repair route + * recomputes only around the repaired files). */ +static const sc_query_t sc_topology_queries[] = { + {"nodes", "SELECT label,name,qualified_name,file_path,start_line,end_line " + "FROM main.nodes WHERE project=?1 ORDER BY 3,1,4,5,6,2;"}, + {"edges", "SELECT s.label,s.qualified_name,s.file_path,s.start_line,e.type,t.label," + "t.qualified_name,t.file_path,t.start_line,e.url_path_gen,e.local_name_gen " + "FROM main.edges e JOIN main.nodes s ON s.id=e.source_id " + "JOIN main.nodes t ON t.id=e.target_id WHERE e.project=?1 AND e.type NOT IN " + "('SIMILAR_TO','SEMANTICALLY_RELATED') ORDER BY 2,1,3,4,5,7,6,8,9,10,11;"}, + {"file_hashes", "SELECT rel_path,sha256,size FROM main.file_hashes WHERE project=?1 " + "ORDER BY 1,2,3;"}, + {"index_coverage", "SELECT rel_path,kind,detail FROM main.index_coverage WHERE project=?1 " + "ORDER BY 1,2,3;"}, +}; + +static void sc_frame(cbm_sha256_ctx *ctx, unsigned char tag, const void *bytes, uint64_t length) { + unsigned char head[9]; + head[0] = tag; + for (int i = 0; i < 8; i++) { + head[1 + i] = (unsigned char)(length >> (56 - 8 * i)); + } + cbm_sha256_update(ctx, head, sizeof(head)); + if (length) { + cbm_sha256_update(ctx, bytes, (size_t)length); + } +} + +static bool sc_table(sqlite3 *db, const sc_query_t *q, const char *project, cbm_sha256_ctx *ctx) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, q->sql, -1, &st, NULL) != SQLITE_OK) { + return false; + } + bool ok = sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT) == SQLITE_OK; + sc_frame(ctx, 'T', q->name, strlen(q->name)); + uint64_t rows = 0; + int rc = SQLITE_ROW; + while (ok && (rc = sqlite3_step(st)) == SQLITE_ROW) { + int columns = sqlite3_column_count(st); + unsigned char width[4] = {(unsigned char)(columns >> 24), (unsigned char)(columns >> 16), + (unsigned char)(columns >> 8), (unsigned char)columns}; + sc_frame(ctx, 'R', width, sizeof(width)); + for (int c = 0; c < columns; c++) { + switch (sqlite3_column_type(st, c)) { + case SQLITE_NULL: + sc_frame(ctx, 'N', NULL, 0); + break; + case SQLITE_INTEGER: { + char text[32]; + long long value = (long long)sqlite3_column_int64(st, c); + int n = snprintf(text, sizeof(text), "%lld", value); + sc_frame(ctx, 'I', text, (uint64_t)n); + break; + } + case SQLITE_FLOAT: { + char text[64]; + double value = sqlite3_column_double(st, c); + int n = snprintf(text, sizeof(text), "%.17g", value); + sc_frame(ctx, 'F', text, (uint64_t)n); + break; + } + case SQLITE_BLOB: + sc_frame(ctx, 'B', sqlite3_column_blob(st, c), + (uint64_t)sqlite3_column_bytes(st, c)); + break; + default: + sc_frame(ctx, 'S', sqlite3_column_text(st, c), + (uint64_t)sqlite3_column_bytes(st, c)); + break; + } + } + rows++; + } + ok = ok && rc == SQLITE_DONE; + sqlite3_finalize(st); + unsigned char count[8]; + for (int i = 0; i < 8; i++) { + count[i] = (unsigned char)(rows >> (56 - 8 * i)); + } + sc_frame(ctx, 'C', count, sizeof(count)); + return ok; +} + +static int sc_digest(cbm_store_t *s, const char *project, const char *domain, + const sc_query_t *queries, size_t query_count, + unsigned char out[CBM_SHA256_DIGEST_LEN]) { + sqlite3 *db = s && project ? cbm_store_get_db(s) : NULL; + if (!db || !out) { + return CBM_STORE_ERR; + } + cbm_sha256_ctx ctx; + cbm_sha256_init(&ctx); + sc_frame(&ctx, 'D', domain, strlen(domain)); + /* One read transaction, so every table is read from the same snapshot. */ + bool own = sqlite3_get_autocommit(db) != 0; + if (own && sqlite3_exec(db, "BEGIN", NULL, NULL, NULL) != SQLITE_OK) { + return CBM_STORE_ERR; + } + bool ok = true; + for (size_t i = 0; ok && i < query_count; i++) { + ok = sc_table(db, &queries[i], project, &ctx); + } + if (own && sqlite3_exec(db, ok ? "COMMIT" : "ROLLBACK", NULL, NULL, NULL) != SQLITE_OK) { + ok = false; + } + if (!ok) { + return CBM_STORE_ERR; + } + cbm_sha256_final(&ctx, out); + return CBM_STORE_OK; +} + +int cbm_store_graph_content_digest(cbm_store_t *s, const char *project, + unsigned char out[CBM_SHA256_DIGEST_LEN]) { + return sc_digest(s, project, "cbm.graph.content.v1", sc_queries, + sizeof(sc_queries) / sizeof(sc_queries[0]), out); +} + +int cbm_store_graph_topology_digest(cbm_store_t *s, const char *project, + unsigned char out[CBM_SHA256_DIGEST_LEN]) { + return sc_digest(s, project, "cbm.graph.topology.v1", sc_topology_queries, + sizeof(sc_topology_queries) / sizeof(sc_topology_queries[0]), out); +} diff --git a/tests/test_coverage_merge.sh b/tests/test_coverage_merge.sh new file mode 100644 index 000000000..61c1bb818 --- /dev/null +++ b/tests/test_coverage_merge.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# test_coverage_merge.sh — contract of scripts/test-impact/coverage-merge.py, +# the incremental coverage map of a main-branch push: re-run suites come from +# the fresh map verbatim, every other listed suite is carried with its function +# ids remapped by (file, name) onto the new table, a function the new image no +# longer has drops out, an unlisted suite is dropped, and the metadata is +# recounted so the C reader checks it like a full map. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MERGE="$ROOT/scripts/test-impact/coverage-merge.py" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } + +mkdir -p "$TMP/previous" "$TMP/fresh" +# Previous image: f, g, gone. Suite a covers f+g, b covers g+gone, old is gone. +printf '0\tsrc/x.c\tf\n1\tsrc/x.c\tg\n2\tsrc/y.c\tgone\n' > "$TMP/previous/functions.tsv" +printf 'a:*\tcomplete\t\t\na:t1\tcomplete\t\t0 1\nb:*\tcomplete\t\t\nb:t2\tincomplete\tkilled child\t1 2\nold:*\tcomplete\t\t\nold:t3\tcomplete\t\t0\n' \ + > "$TMP/previous/tests.tsv" +cat > "$TMP/previous/meta.json" <<'EOF' +{"format": 1, "commit": "1111111111111111111111111111111111111111", "platform": "p", + "llvm_profdata": "v", "functions": 3, "functions_compiled": 3, "tests": 3, "incomplete": 1, + "suites": [{"suite": "a", "tests": 1, "incomplete": 0, "exit": 0, "wall_s": 1.0}, + {"suite": "b", "tests": 1, "incomplete": 1, "exit": 0, "wall_s": 2.0}, + {"suite": "old", "tests": 1, "incomplete": 0, "exit": 0, "wall_s": 3.0}]} +EOF +# New image: ids shifted, `gone` removed, `h` new. Only suite a re-ran. +printf '0\tsrc/x.c\th\n1\tsrc/x.c\tf\n2\tsrc/x.c\tg\n' > "$TMP/fresh/functions.tsv" +printf 'a:*\tcomplete\t\t\na:t1\tcomplete\t\t0 1\n' > "$TMP/fresh/tests.tsv" +cat > "$TMP/fresh/meta.json" <<'EOF' +{"format": 1, "commit": "2222222222222222222222222222222222222222", "platform": "p", + "llvm_profdata": "v", "functions": 3, "functions_compiled": 3, "tests": 1, "incomplete": 0, + "suites": [{"suite": "a", "tests": 1, "incomplete": 0, "exit": 0, "wall_s": 1.5}]} +EOF +printf 'a\nb\n' > "$TMP/listed.txt" + +python3 "$MERGE" --previous "$TMP/previous" --fresh "$TMP/fresh" --suites "$TMP/listed.txt" \ + --commit 3333333333333333333333333333333333333333 --out "$TMP/out" > /dev/null + +cmp -s "$TMP/out/functions.tsv" "$TMP/fresh/functions.tsv" || fail "the function table is not the new image's" +expected="$(printf 'a:*\tcomplete\t\t\na:t1\tcomplete\t\t0 1\nb:*\tcomplete\t\t\nb:t2\tincomplete\tkilled child\t2\n')" +[ "$(cat "$TMP/out/tests.tsv")" = "$expected" ] || fail "merged rows: $(cat "$TMP/out/tests.tsv")" +python3 - "$TMP/out/meta.json" <<'EOF' || fail "merged metadata" +import json, sys +meta = json.load(open(sys.argv[1])) +assert meta["commit"] == "3" * 40, meta["commit"] +assert (meta["tests"], meta["incomplete"]) == (2, 1), (meta["tests"], meta["incomplete"]) +suites = {s["suite"]: s for s in meta["suites"]} +assert sorted(suites) == ["a", "b"], sorted(suites) +assert (suites["a"]["tests"], suites["a"]["wall_s"]) == (1, 1.5), suites["a"] +assert (suites["b"]["tests"], suites["b"]["incomplete"]) == (1, 1), suites["b"] +EOF + +# A re-run suite the new runner does not list is an inconsistent input. +printf 'b\n' > "$TMP/listed-wrong.txt" +if python3 "$MERGE" --previous "$TMP/previous" --fresh "$TMP/fresh" --suites "$TMP/listed-wrong.txt" \ + --commit 3333333333333333333333333333333333333333 --out "$TMP/out2" > /dev/null 2>&1; then + fail "a re-run suite missing from the runner's list was accepted" +fi +echo "coverage merge: all cases passed" diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index 5e4ef2873..040459c7b 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -304,6 +304,8 @@ TEST(daemon_bootstrap_classifies_stateless_commands_without_client) { char *install[] = {"codebase-memory-mcp", "install", "--dry-run", NULL}; char *uninstall[] = {"codebase-memory-mcp", "uninstall", NULL}; char *update[] = {"codebase-memory-mcp", "update", "-n", NULL}; + char *test_impact[] = {"codebase-memory-mcp", "test-impact", "select", NULL}; + ASSERT_EQ(classify(3, test_impact), CBM_DAEMON_PROCESS_STATELESS); ASSERT_EQ(classify(2, version), CBM_DAEMON_PROCESS_STATELESS); ASSERT_EQ(classify(3, help), CBM_DAEMON_PROCESS_STATELESS); ASSERT_EQ(classify(3, install), CBM_DAEMON_PROCESS_STATELESS); diff --git a/tests/test_main.c b/tests/test_main.c index c2a09e4be..5f1014628 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1734,6 +1734,7 @@ extern void suite_test_impact_classify(void); extern void suite_test_impact_source(void); extern void suite_test_impact_seed(void); extern void suite_test_impact_engine(void); +extern void suite_test_impact_artifact(void); extern void suite_test_impact_runner_filter(void); extern void suite_store_search(void); extern void suite_cypher(void); @@ -2124,6 +2125,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(test_impact_source); RUN_SELECTED_SUITE(test_impact_seed); RUN_SELECTED_SUITE(test_impact_engine); + RUN_SELECTED_SUITE(test_impact_artifact); RUN_SELECTED_SUITE(test_impact_runner_filter); RUN_SELECTED_SUITE(store_search); RUN_SELECTED_SUITE(store_bulk); diff --git a/tests/test_pipeline_frozen.c b/tests/test_pipeline_frozen.c index 028856d7c..c4fe336d1 100644 --- a/tests/test_pipeline_frozen.c +++ b/tests/test_pipeline_frozen.c @@ -1624,6 +1624,89 @@ TEST(pipeline_frozen_rejects_malformed_file_lists) { PASS(); } +/* A base candidate (the team artifact's graph) is repaired incrementally when + * the closure route can, rebuilt when it declines, and either way the result + * is exactly the graph a build from scratch makes of the same tree. */ +/* Two independent builds never share row ids or a database identity, so they + * are compared by graph content. */ +static bool pf_digest(const char *db, unsigned char out[32]) { + cbm_store_t *store = cbm_store_open_path_query(db); + if (!store) + return false; + bool ok = cbm_store_graph_content_digest(store, PF_PROJECT, out) == CBM_STORE_OK; + cbm_store_close(store); + return ok; +} + +/* Builds the source tree into `keep` (renamed from the candidate); base may be + * NULL. *route receives the route that built it. */ +static bool pf_build_into(pf_fixture_t *f, const char *base, const char *keep, + cbm_pipeline_frozen_route_t *route) { + cbm_git_context_t git = pf_git(f); + cbm_pipeline_frozen_inputs_t in = pf_inputs(f, NULL, &git, NULL); + in.base_db_path = base; + cbm_pipeline_frozen_t *owner = NULL; + bool ok = cbm_pipeline_frozen_create(&in, &owner) == CBM_PIPELINE_FROZEN_OK && + pf_build_expected_ok(owner) == CBM_PIPELINE_FROZEN_OK; + *route = cbm_pipeline_frozen_route(owner); + cbm_pipeline_frozen_free(owner); + return ok && rename(f->db, keep) == 0; +} + +TEST(pipeline_frozen_base_repairs_or_rebuilds_to_the_full_graph) { + ASSERT_EQ(pf_init_status, 0); + pf_fixture_t f = {0}; + ASSERT_TRUE(pf_open(&f)); + char base[1024], repaired[1024], full[1024], grown[1024], grown_full[1024]; + ASSERT_TRUE(pf_path(base, sizeof(base), f.home, "base.db")); + ASSERT_TRUE(pf_path(repaired, sizeof(repaired), f.home, "repaired.db")); + ASSERT_TRUE(pf_path(full, sizeof(full), f.home, "full.db")); + ASSERT_TRUE(pf_path(grown, sizeof(grown), f.home, "grown.db")); + ASSERT_TRUE(pf_path(grown_full, sizeof(grown_full), f.home, "grown_full.db")); + ASSERT_TRUE(pf_source_write(&f, "helper.c", "int helper(int x) { return x + 1; }\n")); + ASSERT_TRUE( + pf_source_write(&f, "use.c", "int helper(int x);\nint use(void) { return helper(1); }\n")); + cbm_pipeline_frozen_route_t route = CBM_PIPELINE_FROZEN_ROUTE_INCREMENTAL; + ASSERT_TRUE(pf_build_into(&f, NULL, base, &route)); + ASSERT_EQ(route, CBM_PIPELINE_FROZEN_ROUTE_FULL); + + /* A body edit: the closure route repairs the base. */ + ASSERT_TRUE(pf_source_write(&f, "helper.c", "int helper(int x) { return x + 2; }\n")); + ASSERT_TRUE(pf_build_into(&f, base, repaired, &route)); + ASSERT_EQ(route, CBM_PIPELINE_FROZEN_ROUTE_INCREMENTAL); + ASSERT_TRUE(pf_build_into(&f, NULL, full, &route)); + ASSERT_EQ(route, CBM_PIPELINE_FROZEN_ROUTE_FULL); + unsigned char a[32], b[32], c[32]; + ASSERT_TRUE(pf_digest(repaired, a)); + ASSERT_TRUE(pf_digest(full, b)); + ASSERT_MEM_EQ(a, b, 32); + ASSERT_TRUE(pf_digest(base, c)); /* the digest sees the edit at all */ + ASSERT_TRUE(memcmp(a, c, 32) != 0); + + /* A new definition name: yesterday's graph cannot know its referencers, so + * the route declines and the build is the full one. */ + ASSERT_TRUE(pf_source_write(&f, "helper.c", + "int helper(int x) { return x + 2; }\nint extra(void) { " + "return 3; }\n")); + ASSERT_TRUE(pf_build_into(&f, base, grown, &route)); + ASSERT_EQ(route, CBM_PIPELINE_FROZEN_ROUTE_FULL); + ASSERT_TRUE(pf_build_into(&f, NULL, grown_full, &route)); + ASSERT_TRUE(pf_digest(grown, a)); + ASSERT_TRUE(pf_digest(grown_full, b)); + ASSERT_MEM_EQ(a, b, 32); + + /* The base is read, never written. */ + char again[1024]; + ASSERT_TRUE(pf_path(again, sizeof(again), f.home, "again.db")); + unsigned char before[32], after[32]; + ASSERT_TRUE(pf_digest(base, before)); + ASSERT_TRUE(pf_build_into(&f, base, again, &route)); + ASSERT_TRUE(pf_digest(base, after)); + ASSERT_MEM_EQ(before, after, 32); + ASSERT_TRUE(pf_close(&f)); + PASS(); +} + SUITE(pipeline_frozen) { RUN_TEST(pipeline_frozen_real_parse_gap_is_recorded_not_refused); pf_init_status = cbm_init(); @@ -1641,5 +1724,6 @@ SUITE(pipeline_frozen) { RUN_TEST(pipeline_frozen_indexes_exactly_the_listed_files); RUN_TEST(pipeline_frozen_missing_listed_file_is_input_changed); RUN_TEST(pipeline_frozen_rejects_malformed_file_lists); + RUN_TEST(pipeline_frozen_base_repairs_or_rebuilds_to_the_full_graph); cbm_work_arena_release(); } diff --git a/tests/test_test_impact.c b/tests/test_test_impact.c index 18f649456..35869dafd 100644 --- a/tests/test_test_impact.c +++ b/tests/test_test_impact.c @@ -2500,6 +2500,54 @@ TEST(test_policy_defaults_and_first_match_before_target_resolution) { /* A change to the selection's own configuration runs everything, in every * repository, before any project rule could ignore it (review M-4). */ +/* test_impact.coverage.compatibility_paths: kept in order; strict on keys + * (an ignored misspelling would make admitting a coverage map easier) and on + * paths (repository-relative, no escaping segment). */ +TEST(test_policy_coverage_compatibility_paths_are_strict) { + cbm_test_policy_t *policy = + policy_of("{\"test_impact\":{\"version\":1,\"coverage\":{\"compatibility_paths\":" + "[\"tests/test_framework.h\",\"Makefile.cbm\"]}}}"); + ASSERT_NOT_NULL(policy); + int count = 0; + const char *const *paths = cbm_test_policy_compatibility_paths(policy, &count); + ASSERT_EQ(count, 2); + ASSERT_STR_EQ(paths[0], "tests/test_framework.h"); + ASSERT_STR_EQ(paths[1], "Makefile.cbm"); + cbm_test_policy_free(policy); + + policy = policy_of("{\"test_impact\":{\"version\":1}}"); + ASSERT_NOT_NULL(policy); + ASSERT_NULL(cbm_test_policy_compatibility_paths(policy, &count)); + ASSERT_EQ(count, 0); + cbm_test_policy_free(policy); + + static const char *const refused[] = { + "{\"compatability_paths\":[\"a\"]}", + "{\"compatibility_paths\":[\"a\"],\"extra\":1}", + "{\"compatibility_paths\":\"a\"}", + "{\"compatibility_paths\":[\"/etc/passwd\"]}", + "{\"compatibility_paths\":[\"../outside\"]}", + "{\"compatibility_paths\":[\"a/./b\"]}", + "{\"compatibility_paths\":[\"a//b\"]}", + "{\"compatibility_paths\":[\"dir/\"]}", + "{\"compatibility_paths\":[\"a\\\\b\"]}", + "{\"compatibility_paths\":[\"\"]}", + "{\"compatibility_paths\":[7]}", + "[]", + }; + for (size_t i = 0; i < sizeof(refused) / sizeof(refused[0]); i++) { + char body[256]; + snprintf(body, sizeof(body), "{\"test_impact\":{\"version\":1,\"coverage\":%s}}", + refused[i]); + cbm_test_policy_t *bad = policy_of(body); + if (bad) { + printf(" accepted: %s\n", refused[i]); + } + ASSERT_NULL(bad); + } + PASS(); +} + TEST(test_policy_config_change_runs_all) { cbm_test_policy_t *policy = policy_of("{\"test_impact\":{\"version\":1,\"rules\":[{\"id\":\"late\"," @@ -3697,6 +3745,7 @@ SUITE(test_impact) { RUN_TEST(test_policy_embedded_stars_do_not_cross_or_remove_separators); RUN_TEST(test_policy_retains_exact_snapshot_and_owns_result); RUN_TEST(test_policy_defaults_and_first_match_before_target_resolution); + RUN_TEST(test_policy_coverage_compatibility_paths_are_strict); RUN_TEST(test_policy_config_change_runs_all); RUN_TEST(test_policy_lanes_and_semantic_perf_membership); RUN_TEST(test_policy_globs_braces_escapes_and_directory_boundaries); diff --git a/tests/test_test_impact_artifact.c b/tests/test_test_impact_artifact.c new file mode 100644 index 000000000..c92ebc390 --- /dev/null +++ b/tests/test_test_impact_artifact.c @@ -0,0 +1,168 @@ +/* + * test_test_impact_artifact.c — the team artifact receipt and its digests. + */ +#include "test_framework.h" +#include "test_helpers.h" + +#include "mcp/test_impact_artifact.h" + +#include +#include + +static bool tia_write(const char *dir, const char *name, const char *text) { + char path[1024]; + snprintf(path, sizeof(path), "%s/%s", dir, name); + FILE *f = fopen(path, "wb"); + if (!f) { + return false; + } + bool ok = fputs(text, f) >= 0; + return fclose(f) == 0 && ok; +} + +static cbm_ti_receipt_t tia_receipt(bool coverage) { + cbm_ti_receipt_t r = {0}; + memset(r.commit, 'a', 40); + memset(r.graph_sha256, 'b', 64); + memset(r.graph_content_sha256, '9', 64); + memset(r.graph_topology_sha256, '8', 64); + snprintf(r.platform, sizeof(r.platform), "linux-x86_64-clang"); + r.has_coverage = coverage; + if (coverage) { + memset(r.functions_sha256, 'c', 64); + memset(r.tests_sha256, 'd', 64); + memset(r.metadata_sha256, 'e', 64); + memset(r.compatibility_sha256, 'f', 64); + r.oldest_observation_at = 1791150000; + } + return r; +} + +TEST(test_impact_artifact_receipt_round_trips) { + char *dir = th_mktempdir("cbm_tia_rt"); + ASSERT_NOT_NULL(dir); + for (int coverage = 0; coverage < 2; coverage++) { + cbm_ti_receipt_t in = tia_receipt(coverage); + ASSERT_TRUE(cbm_ti_receipt_write(dir, &in)); + cbm_ti_receipt_t out; + ASSERT_EQ(cbm_ti_receipt_read(dir, &out), CBM_TI_RECEIPT_OK); + ASSERT_STR_EQ(out.commit, in.commit); + ASSERT_STR_EQ(out.graph_sha256, in.graph_sha256); + ASSERT_STR_EQ(out.graph_content_sha256, in.graph_content_sha256); + ASSERT_STR_EQ(out.graph_topology_sha256, in.graph_topology_sha256); + ASSERT_STR_EQ(out.platform, in.platform); + ASSERT_EQ(out.has_coverage, in.has_coverage); + ASSERT_STR_EQ(out.functions_sha256, in.functions_sha256); + ASSERT_STR_EQ(out.compatibility_sha256, in.compatibility_sha256); + ASSERT_EQ(out.oldest_observation_at, in.oldest_observation_at); + } + th_rmtree(dir); + PASS(); +} + +TEST(test_impact_artifact_receipt_is_strict) { + char *dir = th_mktempdir("cbm_tia_strict"); + ASSERT_NOT_NULL(dir); + cbm_ti_receipt_t out; + ASSERT_EQ(cbm_ti_receipt_read(dir, &out), CBM_TI_RECEIPT_ABSENT); + static const char *const bad[] = { + "not json", + "{}", + /* unknown schema */ + "{\"schema\":\"cbm.test_impact.artifact.v2\",\"commit\":\"" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"graph_sha256\":\"" + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"graph_content_" + "sha256\":\"9999999999999999999999999999999999999999999999999999999999999999\"," + "\"graph_topology_sha256\":" + "\"8888888888888888888888888888888888888888888888888888888888888888\",\"platform\":\"p\"," + "\"coverage\":null}", + /* a commit of the wrong length */ + "{\"schema\":\"cbm.test_impact.artifact.v1\",\"commit\":\"abc\",\"graph_sha256\":\"" + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"graph_content_" + "sha256\":\"9999999999999999999999999999999999999999999999999999999999999999\"," + "\"graph_topology_sha256\":" + "\"8888888888888888888888888888888888888888888888888888888888888888\",\"platform\":\"p\"," + "\"coverage\":null}", + /* uppercase digest */ + "{\"schema\":\"cbm.test_impact.artifact.v1\",\"commit\":\"" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"graph_sha256\":\"" + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\",\"graph_content_" + "sha256\":\"9999999999999999999999999999999999999999999999999999999999999999\"," + "\"graph_topology_sha256\":" + "\"8888888888888888888888888888888888888888888888888888888888888888\",\"platform\":\"p\"," + "\"coverage\":null}", + /* coverage without its observation time */ + "{\"schema\":\"cbm.test_impact.artifact.v1\",\"commit\":\"" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"graph_sha256\":\"" + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"graph_content_" + "sha256\":\"9999999999999999999999999999999999999999999999999999999999999999\"," + "\"graph_topology_sha256\":" + "\"8888888888888888888888888888888888888888888888888888888888888888\",\"platform\":\"p\"," + "\"coverage\":{\"functions_sha256\":\"" + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"tests_sha256\":\"" + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"metadata_sha256\":\"" + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"," + "\"compatibility_sha256\":\"" + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"}}", + }; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { + ASSERT_TRUE(tia_write(dir, CBM_TI_RECEIPT_FILE, bad[i])); + ASSERT_EQ(cbm_ti_receipt_read(dir, &out), CBM_TI_RECEIPT_INVALID); + ASSERT_EQ(out.commit[0], '\0'); + } + th_rmtree(dir); + PASS(); +} + +TEST(test_impact_artifact_compatibility_digest_binds_every_input) { + char *dir = th_mktempdir("cbm_tia_compat"); + ASSERT_NOT_NULL(dir); + ASSERT_TRUE(tia_write(dir, "harness.h", "int x;\n")); + const char *paths[] = {"harness.h", "missing.mk"}; + char base[65], again[65], other[65]; + ASSERT_TRUE(cbm_ti_compatibility_digest(dir, paths, 2, "linux", base)); + ASSERT_TRUE(cbm_ti_compatibility_digest(dir, paths, 2, "linux", again)); + ASSERT_STR_EQ(base, again); + /* another platform */ + ASSERT_TRUE(cbm_ti_compatibility_digest(dir, paths, 2, "darwin", other)); + ASSERT_TRUE(strcmp(base, other) != 0); + /* a byte of a listed file */ + ASSERT_TRUE(tia_write(dir, "harness.h", "int y;\n")); + ASSERT_TRUE(cbm_ti_compatibility_digest(dir, paths, 2, "linux", other)); + ASSERT_TRUE(strcmp(base, other) != 0); + ASSERT_TRUE(tia_write(dir, "harness.h", "int x;\n")); + /* an absent listed file appearing, even empty */ + ASSERT_TRUE(tia_write(dir, "missing.mk", "")); + ASSERT_TRUE(cbm_ti_compatibility_digest(dir, paths, 2, "linux", other)); + ASSERT_TRUE(strcmp(base, other) != 0); + /* the list itself */ + ASSERT_TRUE(cbm_ti_compatibility_digest(dir, paths, 1, "linux", other)); + ASSERT_TRUE(strcmp(base, other) != 0); + /* an empty listed path is refused, not hashed */ + const char *empty[] = {""}; + ASSERT_FALSE(cbm_ti_compatibility_digest(dir, empty, 1, "linux", other)); + th_rmtree(dir); + PASS(); +} + +TEST(test_impact_artifact_file_digest_is_exact_bytes) { + char *dir = th_mktempdir("cbm_tia_sha"); + ASSERT_NOT_NULL(dir); + ASSERT_TRUE(tia_write(dir, "f", "abc")); + char path[1024]; + snprintf(path, sizeof(path), "%s/f", dir); + char hex[65]; + ASSERT_TRUE(cbm_ti_sha256_file(path, hex)); + ASSERT_STR_EQ(hex, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + snprintf(path, sizeof(path), "%s/absent", dir); + ASSERT_FALSE(cbm_ti_sha256_file(path, hex)); + th_rmtree(dir); + PASS(); +} + +SUITE(test_impact_artifact) { + RUN_TEST(test_impact_artifact_receipt_round_trips); + RUN_TEST(test_impact_artifact_receipt_is_strict); + RUN_TEST(test_impact_artifact_compatibility_digest_binds_every_input); + RUN_TEST(test_impact_artifact_file_digest_is_exact_bytes); +} diff --git a/tests/test_test_impact_engine.c b/tests/test_test_impact_engine.c index 8dbaa2d14..7bae6388b 100644 --- a/tests/test_test_impact_engine.c +++ b/tests/test_test_impact_engine.c @@ -19,6 +19,7 @@ #include #include #include +#include typedef struct { char git[1024]; @@ -152,12 +153,18 @@ static void tie_close(tie_fixture_t *fx) { (void)th_rmtree(fx->home); } -/* The answer for the topic branch against main, as JSON (owned). */ -static char *tie_answer(tie_fixture_t *fx) { +/* The answer for the topic branch against main, as JSON (owned); with a + * team artifact bundle when one is given. */ +static char *tie_answer_with(tie_fixture_t *fx, const char *bundle, const char *commit, + bool verified) { cbm_test_impact_request_t rq = {.repo_root = fx->repo, .base_ref = "main", .work_parent = fx->work, - .deadline_ms = cbm_now_ms() + 120000U}; + .deadline_ms = cbm_now_ms() + 120000U, + .artifact_dir = bundle, + .artifact_commit = commit, + .artifact_verified = verified, + .platform = "fixture"}; cbm_test_result_t *out = NULL; char diagnostic[512]; if (cbm_test_impact_run(&rq, &out, diagnostic, sizeof(diagnostic)) != CBM_TEST_IMPACT_OK) { @@ -174,6 +181,10 @@ static char *tie_answer(tie_fixture_t *fx) { return copy; } +static char *tie_answer(tie_fixture_t *fx) { + return tie_answer_with(fx, NULL, NULL, false); +} + /* The "reasons" array of one suite's entry in the answer. */ static bool tie_suite_has(const char *json, const char *suite, const char *reason) { char key[128]; @@ -297,9 +308,159 @@ TEST(test_impact_engine_parse_gaps_reach_what_they_name) { PASS(); } +/* The base commit's object id, read from the branch ref (never packed here). */ +static bool tie_base_sha(tie_fixture_t *fx, char sha[65]) { + char path[1200]; + snprintf(path, sizeof(path), "%s/.git/refs/heads/main", fx->repo); + FILE *f = fopen(path, "rb"); + if (!f) { + return false; + } + size_t n = fread(sha, 1, 64, f); + (void)fclose(f); + while (n && (sha[n - 1] == '\n' || sha[n - 1] == '\r')) { + n--; + } + sha[n] = '\0'; + return n == 40 || n == 64; +} + +/* A coverage map of the base: lib_value runs only under alpha_uses_lib, + * lib_other only under matrix_one; every setup row is complete and empty. */ +static bool tie_coverage(tie_fixture_t *fx, const char *sha, char *dir, size_t cap) { + snprintf(dir, cap, "%s/coverage", fx->home); + char meta[1024]; + snprintf(meta, sizeof(meta), + "{\"format\":1,\"commit\":\"%s\",\"functions\":2,\"tests\":4," + "\"incomplete\":0,\"platform\":\"fixture\",\"llvm_profdata\":\"fixture\"," + "\"suites\":[{\"suite\":\"alpha\",\"tests\":2,\"incomplete\":0,\"exit\":0}," + "{\"suite\":\"beta\",\"tests\":1,\"incomplete\":0,\"exit\":0}," + "{\"suite\":\"matrix\",\"tests\":1,\"incomplete\":0,\"exit\":0}]}\n", + sha); + return th_mkdir_p(dir) == 0 && + th_write_file(TH_PATH(dir, "functions.tsv"), + "0\tsrc/lib.c\tlib_value\n1\tsrc/lib.c\tlib_other\n") == 0 && + th_write_file(TH_PATH(dir, "tests.tsv"), + "alpha:*\tcomplete\t\t\nalpha:alpha_plain\tcomplete\t\t\n" + "alpha:alpha_uses_lib\tcomplete\t\t0\nbeta:*\tcomplete\t\t\n" + "beta:beta_alone\tcomplete\t\t\nmatrix:*\tcomplete\t\t\n" + "matrix:matrix_one\tcomplete\t\t1\n") == 0 && + th_write_file(TH_PATH(dir, "meta.json"), meta) == 0; +} + +/* The team artifact: the merge base's bundle, verified by the caller, is the + * graph base and its admitted map narrows a suite to the tests that ran the + * changed function; the same bundle unverified is ignored, and without a + * bundle every suite stays whole. */ +TEST(test_impact_engine_admitted_coverage_narrows_only_when_verified) { + tie_fixture_t fx; + ASSERT_TRUE(tie_open(&fx)); + char sha[65]; + char coverage[1200]; + char bundle[1200]; + ASSERT_TRUE(tie_base_sha(&fx, sha)); + ASSERT_TRUE(tie_coverage(&fx, sha, coverage, sizeof(coverage))); + snprintf(bundle, sizeof(bundle), "%s/bundle", fx.home); + cbm_test_impact_publish_t publish = {.repo_root = fx.repo, + .work_parent = fx.work, + .out_dir = bundle, + .coverage_dir = coverage, + .observed_at = (int64_t)time(NULL), + .platform = "fixture", + .deadline_ms = cbm_now_ms() + 120000U}; + char diagnostic[512] = ""; + cbm_test_impact_status_t published = + cbm_test_impact_publish(&publish, diagnostic, sizeof(diagnostic)); + if (published != CBM_TEST_IMPACT_OK) { + printf(" publish: %s\n", diagnostic); + } + ASSERT_EQ(published, CBM_TEST_IMPACT_OK); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1 + 0;\n}\n" + "int lib_other(void) {\n return 2;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_value")); + + char *plain = tie_answer(&fx); + char *verified = tie_answer_with(&fx, bundle, sha, true); + char *unverified = tie_answer_with(&fx, bundle, sha, false); + ASSERT_NOT_NULL(plain); + ASSERT_NOT_NULL(verified); + ASSERT_NOT_NULL(unverified); + if (!strstr(verified, "\"mode\":\"tests\",\"suite\":\"alpha\"")) { + printf(" %.1200s\n", verified); + } + /* Narrowed: alpha to the one test that ran lib_value; beta not at all. */ + ASSERT_NOT_NULL(strstr(verified, "\"mode\":\"tests\",\"suite\":\"alpha\"")); + ASSERT_NOT_NULL(strstr(verified, "\"test\":\"alpha_uses_lib\"")); + ASSERT_NULL(strstr(verified, "\"test\":\"alpha_plain\"")); + ASSERT_NULL(strstr(verified, "\"suite\":\"beta\"")); + /* Unverified and absent: no narrowing. */ + ASSERT_NOT_NULL(strstr(unverified, "\"mode\":\"whole\",\"suite\":\"beta\"")); + ASSERT_NOT_NULL(strstr(unverified, "SOURCE_UNVERIFIED")); + ASSERT_NOT_NULL(strstr(plain, "\"mode\":\"whole\",\"suite\":\"beta\"")); + free(plain); + free(verified); + free(unverified); + tie_close(&fx); + PASS(); +} + +/* A changed function the map does not know (here: not in the map's image, + * as platform code would not be) reaches the coverage of its nearest mapped + * callers, so the tests that ran lib_value are selected BY COVERAGE when only + * lib_win changed. */ +static const char tie_lib_win[] = "int lib_win(void) {\n" + " return 0;\n" + "}\n" + "int lib_value(void) {\n" + " return 1 + lib_win();\n" + "}\n" + "int lib_other(void) {\n" + " return 2;\n" + "}\n"; + +TEST(test_impact_engine_coverage_reaches_unmapped_code_through_mapped_callers) { + tie_fixture_t fx; + ASSERT_TRUE(tie_open_with(&fx, "src/lib.c", tie_lib_win)); + char sha[65]; + char coverage[1200]; + char bundle[1200]; + ASSERT_TRUE(tie_base_sha(&fx, sha)); + ASSERT_TRUE(tie_coverage(&fx, sha, coverage, sizeof(coverage))); + snprintf(bundle, sizeof(bundle), "%s/bundle", fx.home); + cbm_test_impact_publish_t publish = {.repo_root = fx.repo, + .work_parent = fx.work, + .out_dir = bundle, + .coverage_dir = coverage, + .observed_at = (int64_t)time(NULL), + .platform = "fixture", + .deadline_ms = cbm_now_ms() + 120000U}; + char diagnostic[512] = ""; + ASSERT_EQ(cbm_test_impact_publish(&publish, diagnostic, sizeof(diagnostic)), + CBM_TEST_IMPACT_OK); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_win(void) {\n return 0 + 0;\n}\n" + "int lib_value(void) {\n return 1 + lib_win();\n}\n" + "int lib_other(void) {\n return 2;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_win only")); + char *verified = tie_answer_with(&fx, bundle, sha, true); + ASSERT_NOT_NULL(verified); + const char *covered = "\"id\":\"alpha:alpha_uses_lib\",\"line\":2,\"reasons\":[\"COVERAGE\""; + if (!strstr(verified, covered)) { + printf(" %.1200s\n", verified); + } + ASSERT_NOT_NULL(strstr(verified, covered)); + ASSERT_NULL(strstr(verified, "\"test\":\"alpha_plain\"")); + free(verified); + tie_close(&fx); + PASS(); +} + SUITE(test_impact_engine) { RUN_TEST(test_impact_engine_reached_suites_carry_static); RUN_TEST(test_impact_engine_macro_registered_tests_select_their_suite); RUN_TEST(test_impact_engine_rules_unmapped_and_empty); RUN_TEST(test_impact_engine_parse_gaps_reach_what_they_name); + RUN_TEST(test_impact_engine_admitted_coverage_narrows_only_when_verified); + RUN_TEST(test_impact_engine_coverage_reaches_unmapped_code_through_mapped_callers); }