fix(discover): a .gitignore never hides a file git tracks

git applies ignore rules to untracked paths only, but discovery applied
them to every path. A nested .gitignore line `manager`, meant for a
built binary, hid cluster-api's tracked config/manager/ kustomization,
and with it the Deployment its overlays point at.

When the indexed directory is in a git repository, discovery now reads
`git ls-files -z` once. A directory that git's rules ignore (.gitignore
files, info/exclude, the global excludes file) is still entered when it
holds tracked files. Inside it, untracked files and directories stay
ignored, as in git. A tracked file is kept although a rule matches it.
The built-in skip list, .cbmignore and the size cap apply as before. If
git cannot run, nothing changes.

On the 17 infra corpora git lists 34 tracked files that an ignore rule
matches. The fix indexes 26 of them, all from that list. The other 8 are
never indexed for other reasons: no language (.tfvars, .gitkeep) or a
built-in skip directory (vendor/). No file is lost and no untracked
file appears. cluster-api gains the manager kustomization (4 Resources,
its overlay IMPORTS), helm-charts a Chart, and otel-demo a Route.

Test: discover_gitignore_never_hides_tracked_files builds a real git
repository. It fails without the fix.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

src/discover/discover.c

@@ -15,8 +15,10 @@
 
 #include "foundation/constants.h"
 #include "foundation/compat_fs.h"
+#include "foundation/hash_table.h"
 #include "foundation/limits.h"
 #include "foundation/mem_core.h"
+#include "foundation/str_util.h"
 #include "foundation/workspace.h"
 #include "foundation/platform.h"
 #ifdef _WIN32
@@ -436,6 +438,25 @@ bool cbm_matches_fast_pattern(const char *filename, cbm_index_mode_t mode) {
 
 /* ── Dynamic file list ────────────────────────── */
 
+/* Paths git tracks under the discovery root, and every directory above one.
+ * git applies ignore rules to untracked paths only, so .gitignore,
+ * info/exclude and the global excludes file never hide a tracked path. Both
+ * tables stay NULL when the root is not in a git repository or git cannot
+ * run, and the ignore rules then apply exactly as before. */
+typedef struct {
+    char *paths;         /* `git ls-files -z` output; holds the keys of `files` */
+    CBMHashTable *files; /* tracked path */
+    CBMHashTable *dirs;  /* directory above a tracked path (owned key) */
+} discover_tracked_t;
+
+static bool tracked_has_file(const discover_tracked_t *tracked, const char *rel_path) {
+    return tracked && tracked->files && cbm_ht_has(tracked->files, rel_path);
+}
+
+static bool tracked_has_dir(const discover_tracked_t *tracked, const char *rel_path) {
+    return tracked && tracked->dirs && cbm_ht_has(tracked->dirs, rel_path);
+}
+
 typedef struct {
     cbm_file_info_t *files;
     int count;
@@ -465,6 +486,7 @@ typedef struct {
     int ignored_count;
     int ignored_cap;
     int ignored_total;
+    const discover_tracked_t *tracked;
 } file_list_t;
 
 static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource,
@@ -765,11 +787,16 @@ static bool dir_is_cache_tree(const char *abs_path) {
     return abs_path[n] == '\0' || abs_path[n] == '/' || abs_path[n] == '\\';
 }
 
-static bool should_skip_directory(const char *entry_name, const char *rel_path,
-                                  const cbm_discover_opts_t *opts,
-                                  const gitignore_link_t *ignore_chain,
-                                  const cbm_gitignore_t *global_gi,
-                                  const cbm_gitignore_t *cbmignore) {
+/* What the ignore rules say about a directory. DIR_GIT_IGNORED comes from
+ * git's own rules (.gitignore files, info/exclude, the global excludes file),
+ * which yield to tracked content; built-in skips and .cbmignore never do. */
+typedef enum { DIR_KEEP, DIR_SKIP, DIR_GIT_IGNORED } dir_verdict_t;
+
+static dir_verdict_t directory_verdict(const char *entry_name, const char *rel_path,
+                                       const cbm_discover_opts_t *opts,
+                                       const gitignore_link_t *ignore_chain,
+                                       const cbm_gitignore_t *global_gi,
+                                       const cbm_gitignore_t *cbmignore) {
     if (cbm_should_skip_dir(entry_name, opts ? opts->mode : CBM_MODE_FULL) ||
         cbm_discover_path_has_skip_suffix(rel_path)) {
         /* #500: a .cbmignore negation (e.g. "!obj/") whose rule is the last
@@ -779,35 +806,36 @@ static bool should_skip_directory(const char *entry_name, const char *rel_path,
         bool unskipped = cbmignore && !cbm_discover_is_safety_core_directory(entry_name) &&
                          cbm_gitignore_match_result(cbmignore, rel_path, true) < 0;
         if (!unskipped) {
-            return true;
+            return DIR_SKIP;
         }
     }
-    if (gitignore_chain_result(ignore_chain, rel_path, true) > 0) {
-        return true;
-    }
+    bool git_ignored = gitignore_chain_result(ignore_chain, rel_path, true) > 0;
     bool global_ignored = global_gi && cbm_gitignore_matches(global_gi, rel_path, true);
     if (cbmignore) {
         int cbm_result = cbm_gitignore_match_result(cbmignore, rel_path, true);
         if (cbm_result > 0) {
-            return true;
+            return DIR_SKIP;
         }
-        if (cbm_result < 0 && global_ignored) {
-            return false;
+        if (cbm_result < 0) {
+            global_ignored = false; /* a .cbmignore negation un-ignores a global match */
         }
     }
-    return global_ignored;
+    return git_ignored || global_ignored ? DIR_GIT_IGNORED : DIR_KEEP;
 }
 
 /* Check if a regular file should be skipped (filters + gitignore + size). */
 /* Classify why a file is skipped. Returns a static reason string (the #963
  * "purposely not indexed" class) or NULL to keep the file. Check order and
  * semantics — including the .cbmignore negation un-ignoring a global-gitignore
- * match — are IDENTICAL to the original boolean predicate. */
+ * match — are IDENTICAL to the original boolean predicate. git's own rules
+ * never hide a tracked file; inside a git-ignored directory (entered only for
+ * its tracked content) every untracked file is ignored, as in git. */
 static const char *file_skip_reason(const char *entry_name, const char *rel_path,
                                     const cbm_discover_opts_t *opts,
                                     const gitignore_link_t *ignore_chain,
                                     const cbm_gitignore_t *global_gi,
-                                    const cbm_gitignore_t *cbmignore, off_t file_size) {
+                                    const cbm_gitignore_t *cbmignore, off_t file_size, bool tracked,
+                                    bool in_ignored_dir) {
     cbm_index_mode_t mode = opts ? opts->mode : CBM_MODE_FULL;
     if (cbm_has_ignored_suffix(entry_name, mode)) {
         return "ignored-suffix";
@@ -818,10 +846,11 @@ static const char *file_skip_reason(const char *entry_name, const char *rel_path
     if (cbm_matches_fast_pattern(entry_name, mode)) {
         return "fast-pattern";
     }
-    if (gitignore_chain_result(ignore_chain, rel_path, false) > 0) {
+    if (!tracked && (in_ignored_dir || gitignore_chain_result(ignore_chain, rel_path, false) > 0)) {
         return "gitignore";
     }
-    bool global_ignored = global_gi && cbm_gitignore_matches(global_gi, rel_path, false);
+    bool global_ignored =
+        !tracked && global_gi && cbm_gitignore_matches(global_gi, rel_path, false);
     if (cbmignore) {
         int cbm_result = cbm_gitignore_match_result(cbmignore, rel_path, false);
         if (cbm_result > 0) {
@@ -1049,9 +1078,10 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co
                                   const gitignore_link_t *ignore_chain,
                                   const cbm_gitignore_t *global_gi,
                                   const cbm_gitignore_t *cbmignore, const sphinx_suffixes_t *sphinx,
-                                  const struct stat *st, file_list_t *out) {
+                                  bool in_ignored_dir, const struct stat *st, file_list_t *out) {
     const char *skip_reason =
-        file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size);
+        file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size,
+                         tracked_has_file(out->tracked, rel_path), in_ignored_dir);
     if (skip_reason) {
         /* Deliberately not indexed (#963) — record so callers can surface it.
          * Unsupported-language files below are NOT recorded: "no grammar for
@@ -1075,6 +1105,7 @@ typedef struct {
     char prefix[CBM_SZ_4K];
     const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */
     sphinx_suffixes_t sphinx;             /* the documentation set's extra reST suffixes */
+    bool git_ignored; /* ignored by git's rules, entered only for its tracked content */
 } walk_frame_t;
 /* Initial capacity only — the stack grows on demand. A single directory can
  * hold more pending sibling frames than any fixed cap (dotnet/runtime has 855
@@ -1112,7 +1143,7 @@ static cbm_gitignore_t *try_load_nested_gitignore(const walk_frame_t *frame) {
  * context. Grows the stack geometrically; the caller's `parent` must not
  * point into the stack array (walk_dir pops into a local copy). */
 static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char *rel_path,
-                             const walk_frame_t *parent, file_list_t *out) {
+                             const walk_frame_t *parent, bool git_ignored, file_list_t *out) {
     if (ws->top >= ws->cap) {
         int new_cap = ws->cap * 2;
         walk_frame_t *grown = realloc(ws->frames, (size_t)new_cap * sizeof(*grown));
@@ -1133,6 +1164,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char
     }
     slot->ignore_chain = parent->ignore_chain;
     slot->sphinx = parent->sphinx;
+    slot->git_ignored = git_ignored;
     ws->top++;
 }
 
@@ -1171,17 +1203,23 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram
     }
 
     if (S_ISDIR(st.st_mode)) {
-        if (!dir_is_cache_tree(abs_path) &&
-            !should_skip_directory(entry->name, rel_path, opts, frame->ignore_chain, global_gi,
-                                   cbmignore)) {
-            walk_push_subdir(ws, abs_path, rel_path, frame, out);
+        dir_verdict_t verdict = dir_is_cache_tree(abs_path)
+                                    ? DIR_SKIP
+                                    : directory_verdict(entry->name, rel_path, opts,
+                                                        frame->ignore_chain, global_gi, cbmignore);
+        if (verdict == DIR_KEEP && frame->git_ignored) {
+            verdict = DIR_GIT_IGNORED; /* below a git-ignored directory, as in git */
+        }
+        if (verdict == DIR_KEEP ||
+            (verdict == DIR_GIT_IGNORED && tracked_has_dir(out->tracked, rel_path))) {
+            walk_push_subdir(ws, abs_path, rel_path, frame, verdict == DIR_GIT_IGNORED, out);
         } else {
             /* Record the excluded subtree root so callers can report it (#411). */
             file_list_add_excluded(out, rel_path);
         }
     } else if (S_ISREG(st.st_mode)) {
         walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi,
-                              cbmignore, &frame->sphinx, &st, out);
+                              cbmignore, &frame->sphinx, frame->git_ignored, &st, out);
     }
 }
 
@@ -1315,6 +1353,124 @@ static bool discover_path_is_absolute(const char *path) {
 #endif
 }
 
+static void tracked_free_key(const char *key, void *value, void *userdata) {
+    (void)value;
+    (void)userdata;
+    cbm_free(CBM_MEM_CLASS_OTHER, (void *)key);
+}
+
+static void tracked_free(discover_tracked_t *tracked) {
+    if (tracked->dirs) {
+        cbm_ht_foreach(tracked->dirs, tracked_free_key, NULL);
+    }
+    cbm_ht_free(tracked->dirs);
+    cbm_ht_free(tracked->files);
+    cbm_free(CBM_MEM_CLASS_OTHER, tracked->paths);
+    memset(tracked, 0, sizeof(*tracked));
+}
+
+/* The whole `git ls-files -z` output for repo_path (paths relative to it),
+ * NUL-terminated; NULL when git cannot run or fails. */
+static char *tracked_read(const char *repo_path, size_t *length) {
+    *length = 0;
+    if (!cbm_validate_shell_path_arg(repo_path)) {
+        return NULL;
+    }
+#ifdef _WIN32
+    const char *null_dev = "NUL";
+#else
+    const char *null_dev = "/dev/null";
+#endif
+    char cmd[CBM_SZ_4K + CBM_SZ_64];
+    int n = snprintf(cmd, sizeof(cmd), "git -C \"%s\" ls-files -z 2>%s", repo_path, null_dev);
+    if (n < 0 || (size_t)n >= sizeof(cmd)) {
+        return NULL;
+    }
+    FILE *fp = cbm_popen_git(cmd);
+    if (!fp) {
+        return NULL;
+    }
+    size_t capacity = CBM_SZ_64K;
+    size_t used = 0;
+    char *buffer = cbm_alloc(CBM_MEM_CLASS_OTHER, capacity);
+    while (buffer) {
+        if (capacity - used < CBM_SZ_4K) {
+            char *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, buffer, capacity * 2);
+            if (!grown) {
+                cbm_free(CBM_MEM_CLASS_OTHER, buffer);
+                buffer = NULL;
+                break;
+            }
+            buffer = grown;
+            capacity *= 2;
+        }
+        size_t got = fread(buffer + used, 1, capacity - used - SKIP_ONE, fp);
+        if (got == 0) {
+            break;
+        }
+        used += got;
+    }
+    if (cbm_pclose(fp) != 0 || !buffer) {
+        cbm_free(CBM_MEM_CLASS_OTHER, buffer);
+        return NULL;
+    }
+    buffer[used] = '\0';
+    *length = used;
+    return buffer;
+}
+
+/* Record every directory above `path` (its '/'-separated prefixes). */
+static bool tracked_add_dirs(CBMHashTable *dirs, const char *path) {
+    char prefix[CBM_SZ_4K];
+    for (const char *slash = strchr(path, '/'); slash; slash = strchr(slash + SKIP_ONE, '/')) {
+        size_t n = (size_t)(slash - path);
+        if (n == 0 || n >= sizeof(prefix)) {
+            continue;
+        }
+        memcpy(prefix, path, n);
+        prefix[n] = '\0';
+        if (cbm_ht_has(dirs, prefix)) {
+            continue;
+        }
+        char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, prefix);
+        if (!key) {
+            return false;
+        }
+        cbm_ht_set(dirs, key, key);
+    }
+    return true;
+}
+
+/* Load what git tracks under repo_path; leaves `tracked` empty on any failure,
+ * so the ignore rules then apply exactly as they did without it. */
+static void tracked_load(discover_tracked_t *tracked, const char *repo_path) {
+    memset(tracked, 0, sizeof(*tracked));
+    size_t length = 0;
+    tracked->paths = tracked_read(repo_path, &length);
+    if (!tracked->paths || length == 0) {
+        tracked_free(tracked);
+        return;
+    }
+    tracked->files = cbm_ht_create(CBM_SZ_1K);
+    tracked->dirs = cbm_ht_create(CBM_SZ_256);
+    if (!tracked->files || !tracked->dirs) {
+        tracked_free(tracked);
+        return;
+    }
+    for (size_t at = 0; at < length;) {
+        const char *path = tracked->paths + at;
+        size_t n = strlen(path);
+        if (n > 0) {
+            cbm_ht_set(tracked->files, path, (void *)path);
+            if (!tracked_add_dirs(tracked->dirs, path)) {
+                tracked_free(tracked);
+                return;
+            }
+        }
+        at += n + SKIP_ONE;
+    }
+}
+
 /* Resolve the shared "common" git directory for repo_path.
  * Handles three layouts:
  *   1. <repo>/.git is a directory    - ordinary repo; common_dir == <repo>/.git
@@ -1679,10 +1835,16 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
         .collect_excluded = !count_only && excluded_out != NULL,
         .collect_ignored = !count_only && ignored_out != NULL,
     };
+    discover_tracked_t tracked = {0};
+    if (is_git_repo) {
+        tracked_load(&tracked, repo_path);
+        fl.tracked = &tracked;
+    }
     walk_cache_dir_snapshot();
     walk_dir(repo_path, "", opts, gitignore, ancestors.chain, global_gi, cbmignore, &fl);
 
     /* Cleanup */
+    tracked_free(&tracked);
     ancestor_ignores_free(&ancestors);
     cbm_gitignore_free(gitignore);
     cbm_gitignore_free(global_gi);

tests/test_discover.c

@@ -891,6 +891,76 @@ TEST(discover_gitignore_dir_excluded_issue234) {
     PASS();
 }
 
+static int disc_git(const char *dir, const char *args) {
+    char cmd[1024];
+    snprintf(cmd, sizeof(cmd),
+             "git -C \"%s\" -c user.name=t -c user.email=t@t.io -c init.defaultBranch=master %s",
+             dir, args);
+    return system(cmd);
+}
+
+static bool disc_has(const cbm_file_info_t *files, int count, const char *rel_path) {
+    for (int i = 0; i < count; i++) {
+        if (strcmp(files[i].rel_path, rel_path) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* A .gitignore never hides a file git tracks: git applies ignore rules to
+ * untracked paths only. cluster-api's nested `.gitignore` line `manager`,
+ * meant for a built binary, hid its tracked config/manager/ kustomization.
+ * Untracked ignored files stay out, also inside a directory that is entered
+ * only for its tracked content. */
+TEST(discover_gitignore_never_hides_tracked_files) {
+    char *base = th_mktempdir("cbm_disc_tracked");
+    ASSERT(base != NULL);
+    if (disc_git(base, "init -q") != 0) {
+        th_cleanup(base);
+        FAIL("git init failed");
+    }
+    th_write_file(TH_PATH(base, ".gitignore"), "manager\n*.gen.go\n");
+    th_write_file(TH_PATH(base, "main.go"), "package main\n\nfunc main() {}\n");
+    th_write_file(TH_PATH(base, "config/manager/manager.go"), "package manager\n\nfunc Run() {}\n");
+    th_write_file(TH_PATH(base, "config/manager/scratch.go"), "package manager\n\nfunc S() {}\n");
+    th_write_file(TH_PATH(base, "config/manager/sub/deep.go"), "package sub\n\nfunc Deep() {}\n");
+    th_write_file(TH_PATH(base, "config/manager/tmp/build.go"), "package tmp\n\nfunc B() {}\n");
+    th_write_file(TH_PATH(base, "api/types.gen.go"), "package api\n\nfunc Gen() {}\n");
+    th_write_file(TH_PATH(base, "api/other.gen.go"), "package api\n\nfunc Other() {}\n");
+    int add_rc = disc_git(base, "add .gitignore main.go");
+    int force_rc = disc_git(base, "add -f config/manager/manager.go config/manager/sub/deep.go "
+                                  "api/types.gen.go");
+
+    cbm_discover_opts_t opts = {0};
+    cbm_file_info_t *files = NULL;
+    int count = 0;
+    int rc = cbm_discover(base, &opts, &files, &count);
+    bool main_found = rc == 0 && disc_has(files, count, "main.go");
+    bool manager_found = rc == 0 && disc_has(files, count, "config/manager/manager.go");
+    bool deep_found = rc == 0 && disc_has(files, count, "config/manager/sub/deep.go");
+    bool gen_found = rc == 0 && disc_has(files, count, "api/types.gen.go");
+    bool scratch_found = rc == 0 && disc_has(files, count, "config/manager/scratch.go");
+    bool build_found = rc == 0 && disc_has(files, count, "config/manager/tmp/build.go");
+    bool other_found = rc == 0 && disc_has(files, count, "api/other.gen.go");
+    cbm_discover_free(files, count);
+    th_cleanup(base);
+
+    ASSERT_EQ(add_rc, 0);
+    ASSERT_EQ(force_rc, 0);
+    ASSERT_EQ(rc, 0);
+    ASSERT_TRUE(main_found);
+    /* Tracked: kept although .gitignore matches them or a directory above. */
+    ASSERT_TRUE(manager_found);
+    ASSERT_TRUE(deep_found);
+    ASSERT_TRUE(gen_found);
+    /* Untracked and ignored: still out, also below the entered directory. */
+    ASSERT_FALSE(scratch_found);
+    ASSERT_FALSE(build_found);
+    ASSERT_FALSE(other_found);
+    PASS();
+}
+
 TEST(discover_max_file_size) {
     char *base = th_mktempdir("cbm_disc_size");
     ASSERT(base != NULL);
@@ -2382,6 +2452,7 @@ SUITE(discover) {
     RUN_TEST(discover_missing_global_excludes_is_noop);
     RUN_TEST(discover_cbmignore_negates_global_ignore);
     RUN_TEST(discover_gitignore_dir_excluded_issue234);
+    RUN_TEST(discover_gitignore_never_hides_tracked_files);
     RUN_TEST(discover_max_file_size);
     RUN_TEST(discover_null_path);
     RUN_TEST(discover_nonexistent_path);