codebase-memory-mcp/scripts/ci/select-lanes.sh

#!/usr/bin/env bash
# select-lanes.sh — decide which PR CI lanes a change set must run.
#
# Canonical CI step (called by pr.yml's `changes` job on the PR's file list).
# This is the R1 rule set, measured against every PR push of September 2026
# (767 pushes; scripts/test-impact/replay-selector.sh replays them, plus the 22
# real failures, as a contract): each changed path gets a category, the
# categories give a tier, the tier gives a lane set, and a few categories add
# lanes of their own. `tier` is the classification the analysis measured;
# `full` overrides it with every lane.
#
# It FAILS SAFE: full=true for tier T5, a path no rule knows, an empty file
# list, a truncated one (file_list_truncated: the pulls/files API stops at
# 3000 files), this script, any workflow, the test harness, Makefile.cbm
# (source registration and flag changes look identical in a file list) and
# vendored code. It is a pure function of the file SET (and the expected
# count): order, duplicates, blank lines and CRLF never change a byte of the
# output.
#
# Usage: scripts/ci/select-lanes.sh [--format json|github-output]
#                                   [--expect-files N] [FILE]
#          FILE (default stdin): changed paths, one per line.
#          --expect-files N: the PR's changed_files; fewer distinct paths, or
#          N or the list at the 3000-file API cap, means truncated -> full.
#          json (default): {"full","lanes","reasons","tier"} on one line.
#          github-output: tier=, full=, lanes=, reasons= lines ($GITHUB_OUTPUT).
#        scripts/ci/select-lanes.sh --batch FILE
#          FILE: JSON lines {"id":..., "files":[...], "expect_files"?: N};
#          prints one json result per line with its "id" (the history
#          replay's single process).
#        scripts/ci/select-lanes.sh --list-lanes
#          Every lane name, one per line (the universe ci-ok checks against).
#        scripts/ci/select-lanes.sh --codeql-scope LANGUAGE [FILE]
#          The paths of FILE (default stdin) in that CodeQL language's PR
#          scope (c-cpp, javascript-typescript, python, actions), one per line.
#          The codeql-* lanes are selected by these scopes; the interpreted
#          ones follow .github/codeql/codeql-config.yml (paths/paths-ignore).
# Exit: 0 = decided · 2 = usage error.
set -euo pipefail

usage() { sed -n '2,38p' "$0" | sed 's/^# \{0,1\}//'; }

FORMAT=json
MODE="select"
CODEQL_LANGUAGE=""
INPUT=""
EXPECT=""
while [ $# -gt 0 ]; do
    case "$1" in
    -h | --help)
        usage
        exit 0
        ;;
    --format)
        FORMAT="${2:-}"
        [ $# -gt 1 ] && shift
        ;;
    --format=*) FORMAT="${1#--format=}" ;;
    --batch)
        MODE="batch"
        INPUT="${2:-}"
        [ $# -gt 1 ] && shift
        ;;
    --list-lanes) MODE="list" ;;
    --codeql-scope)
        MODE="codeql-scope"
        CODEQL_LANGUAGE="${2:-}"
        [ $# -gt 1 ] && shift
        ;;
    --expect-files)
        # An empty value (a workflow whose count went missing) must fail
        # loudly, never quietly skip the truncation check.
        EXPECT="${2:-}"
        [ -n "$EXPECT" ] || EXPECT="missing"
        [ $# -gt 1 ] && shift
        ;;
    -*)
        echo "select-lanes.sh: unknown option '$1'. Please consult --help." >&2
        exit 2
        ;;
    *)
        if [ -n "$INPUT" ]; then
            echo "select-lanes.sh: more than one input file. Please consult --help." >&2
            exit 2
        fi
        INPUT="$1"
        ;;
    esac
    shift
done
case "$FORMAT" in
json | github-output) ;;
*)
    echo "select-lanes.sh: --format must be json or github-output. Please consult --help." >&2
    exit 2
    ;;
esac
case "${EXPECT:-0}" in
*[!0-9]*)
    echo "select-lanes.sh: --expect-files takes a file count. Please consult --help." >&2
    exit 2
    ;;
esac
if [ "$MODE" = batch ] && [ -z "$INPUT" ]; then
    echo "select-lanes.sh: --batch needs a file. Please consult --help." >&2
    exit 2
fi
if [ -n "$INPUT" ] && [ ! -f "$INPUT" ]; then
    echo "select-lanes.sh: no such file '$INPUT'. Please consult --help." >&2
    exit 2
fi

# The heredoc below is the program, so stdin input is staged in a file first.
STAGED=""
case "$MODE:$CODEQL_LANGUAGE" in
codeql-scope:c-cpp | codeql-scope:javascript-typescript | codeql-scope:python | codeql-scope:actions | select:* | batch:* | list:*) ;;
*)
    echo "select-lanes.sh: --codeql-scope takes c-cpp, javascript-typescript, python or actions. Please consult --help." >&2
    exit 2
    ;;
esac
if { [ "$MODE" = select ] || [ "$MODE" = codeql-scope ]; } && [ -z "$INPUT" ]; then
    STAGED=$(mktemp "${TMPDIR:-/tmp}/cbm-select-lanes.XXXXXX")
    trap 'rm -f "$STAGED"' EXIT
    cat >"$STAGED"
    INPUT="$STAGED"
fi

CODEQL_CONFIG="$(cd "$(dirname "$0")/../.." && pwd)/.github/codeql/codeql-config.yml"
python3 - "$MODE" "$FORMAT" "$INPUT" "$EXPECT" "$CODEQL_CONFIG" "$CODEQL_LANGUAGE" <<'PY'
import json
import re
import sys

mode, fmt, source = sys.argv[1:4]
expect = int(sys.argv[4]) if sys.argv[4] else None
codeql_config, codeql_language = sys.argv[5:7]

# ── Path -> category (first matching rule wins; order is part of the rule) ──
GATE_WORKFLOWS = {"pr.yml", "_test.yml", "_lint.yml", "_security.yml",
                  "_smoke.yml", "_memwaste.yml", "codeql.yml", "dco.yml"}
BUILD = {"Makefile.cbm", "scripts/test.sh", "scripts/run-tests-parallel.sh",
         "scripts/env.sh", "scripts/build.sh", "scripts/clean.sh",
         "scripts/path-safety.sh", "scripts/test-runtime.sh", "scripts/msan.sh",
         "scripts/lint.sh", ".clang-format", ".clang-tidy",
         "scripts/vendored-checksums.txt", "scripts/run-test-wave.py",
         "scripts/embed-frontend.sh"}
SMOKE = {"scripts/smoke-test.sh", "scripts/smoke-local.sh",
         "scripts/smoke-fixture-server.py", "scripts/smoke-invariants.sh",
         "test-infrastructure/vm/vm-smoke.sh",
         "test-infrastructure/vm/windows-user-path-guard.ps1",
         "scripts/test-windows.ps1", "scripts/smoke-wrappers.sh",
         "scripts/security-fuzz.sh", "scripts/security-fuzz-random.sh",
         "scripts/security-install.sh", "scripts/security-network.sh",
         "scripts/security-strings.sh", "scripts/test_mcp_interactive.py"}
LINT_CONFIG = {"scripts/memory-core-baseline.txt", "scripts/lint-memory-core.py",
               "scripts/hooks/pre-commit", ".cppcheck", "scripts/check-no-test-skips.sh",
               "scripts/check-nolint-whitelist.sh", "scripts/lint-mem-gate.py",
               "scripts/lint-mem-whitelist.txt", "scripts/lint-tidy-diff.sh"}
# Local tools and release-only jobs: no PR lane runs them.
NON_GATING = {"scripts/package-release.sh", "scripts/benchmark-index.sh",
              "scripts/benchmark-search-graph.sh", "scripts/benchmark-streaming.py",
              "scripts/soak-test.sh", ".cbmignore", ".node-version", "flake.nix",
              "flake.lock", "scripts/soak-legs.sh", "scripts/repro.sh",
              "scripts/audit-grammar-security.sh", "scripts/check-dco.sh",
              "scripts/check-lsp-originality.sh", "scripts/clone-bench-repos.sh",
              "scripts/extract_nomic_vectors.py", "scripts/gen-pdf-unicode.py",
              "scripts/gen-py-stdlib.py", "scripts/generate-lang-code.py",
              "scripts/new-languages.json", "scripts/git-hooks/commit-msg",
              "scripts/install-git-hooks.sh", "scripts/sanitizer-coverage-map.py",
              "scripts/sanitizer-map.json", "scripts/test_mcp_rapid_init.py",
              "scripts/vendor-grammar.sh"}
# The license-gate lane runs these (and only that lane).
LICENSE_TOOLS = {"scripts/license-gate.sh", "scripts/license-gate-check.py",
                 "scripts/license-gate-check-npm.py", "scripts/license-policy.json",
                 "scripts/ui_browser_licenses.py"}
# security-static runs these; it runs on every PR (ALWAYS).
SECURITY_TOOLS = {"scripts/security-audit.sh", "scripts/security-allowlist.txt",
                  "scripts/security-network-source-audit.py", "scripts/security-ui.sh"}
# Repository files C tests read: the test legs run (a docs-only tier runs no C
# suite, so test_cli's README/site contract would never see a README change).
READ_BY_TESTS = {"scripts/security-vendored.sh", ".codebase-memory.json", "README.md",
                 "docs/index.html", "docs/llms.txt", "pkg/npm/README.md"}
# The test legs' failure report (tests/test_failure_report_contract.sh).
TEST_REPORT = {"scripts/suite-failure-report.sh"}
# The release notices ship inside every archive.
PKG_SCRIPTS = {"scripts/gen-third-party-notices.sh", "scripts/gen-ui-licenses.py"}
PKG_FILES = {"install.sh", "install.ps1", "server.json", "package.json", "manifest.json",
             "Formula"}
DOC_SUFFIXES = (".md", ".png", ".svg", ".jpg", ".gif", ".txt")
DOC_FILES = {"LICENSE", "CODEOWNERS", ".gitignore", ".gitattributes", "DCO",
             ".gitleaksignore", "glama.json"}


def category(path):
    base = path.rsplit("/", 1)[-1]
    if path == "Makefile.cbm":
        return "build"
    if path in LINT_CONFIG:
        return "lint-cfg"
    if path.startswith(("scripts/memwaste", "scripts/memlab")):
        return "memwaste-tools"
    if path in NON_GATING or path.startswith("scripts/test-impact/"):
        return "ci-other"
    if path == "scripts/test_mcp_in_process.py":
        return "test-contract"
    if path == "scripts/fuzz.sh":
        return "fuzz"
    if path == "scripts/audit-license-provenance.py":
        return "ci-gate"
    if path.startswith(("internal/cbm/vendored/grammars/", "tools/tree-sitter-")):
        return "grammar"
    if path.startswith(".github/workflows/"):
        return "ci-gate" if base in GATE_WORKFLOWS else "ci-other"
    if path.startswith(".github/codeql/"):
        return "ci-gate"
    if path.startswith(".github/"):
        if path.endswith(".md") or ("ISSUE_TEMPLATE" in path and path.endswith((".yml", ".yaml"))):
            return "docs"
        return "ci-other"
    if path.startswith("scripts/ci/"):
        return "ci-gate"
    if path in BUILD:
        return "build"
    if path in SMOKE:
        return "smoke"
    if path in LICENSE_TOOLS:
        return "license-tools"
    if path in SECURITY_TOOLS:
        return "security-tools"
    if path in READ_BY_TESTS:
        return "test"
    if path in TEST_REPORT:
        return "test-contract"
    if path in PKG_SCRIPTS:
        return "pkg"
    if path.startswith("scripts/") and path.endswith(".md"):
        return "docs"
    if path.startswith("tests/fixtures/"):
        return "test"
    if path.startswith("tests/windows/"):
        return "test-win"
    if path.startswith("tests/fuzz/"):
        return "fuzz"
    if path.startswith("tests/"):
        return "test-contract" if path.endswith((".sh", ".py", ".ps1")) else "test"
    if path.startswith(("docs/", "eval-results/")) or base in DOC_FILES or (
            path.endswith(DOC_SUFFIXES) and not path.startswith(("src/", "internal/", "scripts/"))):
        return "docs"
    if path.startswith("graph-ui/"):
        return "ui-frontend"
    if path.startswith(("pkg/", "scripts/setup")) or base in PKG_FILES:
        return "pkg"
    if path.startswith(("internal/cbm/vendored/", "vendored/")):
        return "vendored"
    m = re.match(r"internal/cbm/lsp/([a-z]+)_", path)
    if m and m.group(1) not in ("lsp", "type", "scope"):
        return "lsp:" + m.group(1)
    if path.startswith("internal/cbm/lsp/"):
        return "lsp-core"
    if path.startswith("internal/cbm/"):
        return "extract"
    if path == "src/main.c":
        return "cli"
    if path.startswith("src/foundation/"):
        return "mem-core" if re.search(r"/(mem|slab|arena|alloc|mimalloc)", path) else "foundation"
    if path.startswith("src/"):
        return "src:" + path.split("/")[1]
    if path.startswith("test-infrastructure/"):
        return "local-infra"
    if path.startswith("scripts/"):
        return "scripts-other"
    return "other"


# ── Categories -> tier (highest wins) ──
EXTRACT = {"grammar", "lsp-core", "extract"}
MID = {"pipeline", "mcp", "store", "cypher", "discover", "git", "graph_buffer",
       "semantic", "simhash", "traces", "personal_memory"}
PLATFORM = {"daemon", "cli", "watcher", "ui", "foundation", "smoke", "pkg", "test-win"}
CORE = {"mem-core", "build", "vendored", "ci-gate", "other", "scripts-other"}
LOWER = {"test", "test-contract", "fuzz", "ui-frontend", "ci-other",
         "local-infra", "docs", "lint-cfg", "memwaste-tools", "license-tools",
         "security-tools"}


def normalized(cat):
    # src/<dir>/... categories are the directory name; a directory no tier
    # names stays unknown (and therefore T5 + full).
    if cat.startswith("src:"):
        name = cat[4:]
        return name if name in MID | PLATFORM else cat
    return cat


def known(cat):
    return cat in EXTRACT | MID | PLATFORM | CORE | LOWER or cat.startswith("lsp:")


def tier_of(cats):
    if cats & CORE or not all(known(c) for c in cats):
        return "T5-core"
    if cats & PLATFORM:
        return "T4-platform"
    if cats & MID:
        return "T3-mid"
    if cats & EXTRACT or any(c.startswith("lsp:") for c in cats):
        return "T2-extract"
    if cats & {"test", "test-contract", "fuzz"}:
        return "T1-tests"
    if cats & {"ui-frontend", "ci-other", "local-infra", "lint-cfg", "memwaste-tools",
               "license-tools", "security-tools"}:
        return "T0b-nonproduct"
    return "T0a-docs"


# ── Tier -> lanes (R1) ──
# The PR CodeQL scans, one lane per language (pr.yml's `codeql` matrix). Each
# is selected by a change in its scope (codeql_scope below) and, like every
# lane, by any full run: an undeterminable diff scans every language.
CODEQL_LANES = {"c-cpp": "codeql-cpp", "javascript-typescript": "codeql-js",
                "python": "codeql-py", "actions": "codeql-actions"}
FULL = {"diag", "license-gate", "lint", "lint-mem", "lsan-macos",
        "memwaste", "msan", "pkg-wrappers", "security-static", "shard-completeness",
        "smoke-mac", "smoke-ubuntu", "smoke-win", "tsan-arm", "tsan-mac", "tsan-x86",
        "unix-arm64", "unix-macos-intel", "unix-macos14", "unix-x86", "windows",
        "windows-guards"} | set(CODEQL_LANES.values())
UNIVERSE = FULL | {"contracts"}   # contracts: the T0a/T0b job; every test leg runs them too
LINT = {"lint", "lint-mem"}
CORE_LEGS = {"unix-x86", "unix-macos14", "windows", "shard-completeness"}
TEST_LEGS = {"unix-x86", "unix-arm64", "unix-macos14", "unix-macos-intel", "windows"}
ALWAYS = {"security-static"}
R1 = {
    "T0a-docs": {"contracts"},
    # The contract steps police test-infrastructure/ and lint config too.
    "T0b-nonproduct": LINT | {"contracts"},
    "T1-tests": LINT | CORE_LEGS,
    "T2-extract": LINT | CORE_LEGS | {"tsan-x86", "smoke-ubuntu"},
    "T3-mid": LINT | CORE_LEGS | {"tsan-x86", "smoke-ubuntu"},
    "T4-platform": FULL - {"msan", "diag", "memwaste"} - set(CODEQL_LANES.values()),
    "T5-core": FULL,
}
# Category add-ons: (categories, lanes). pkg adds nothing to a docs-only set
# because a pkg path is never docs-only (it lifts the tier to T4).
ADD_ONS = [
    ({"pkg"}, {"pkg-wrappers", "smoke-ubuntu", "smoke-mac", "smoke-win"}),
    ({"vendored", "pkg", "ci-gate", "build", "license-tools"}, {"license-gate"}),
    ({"ui-frontend"}, {"windows-guards", "smoke-ubuntu"}),
    ({"memwaste-tools", "mem-core"}, {"memwaste"}),
    ({"test-win"}, {"windows", "windows-guards"}),
    # the contract steps run inside test.sh's default leg
    ({"ci-other", "test-contract", "fuzz"}, {"unix-x86"}),
]
HARNESS = {"scripts/test.sh", "scripts/run-tests-parallel.sh", "scripts/run-test-wave.py",
           "tests/test_main.c", "tests/test_framework.h"}


# ── CodeQL PR scope (the codeql-* lanes) ──
# c-cpp: any C/C++ the production build can compile -- first-party AND
# vendored (a vendored-only change is scanned: codeql.yml traces the vendored
# files a PR changes). Tests are not in the analysed build.
# Interpreted languages: their extensions inside the CodeQL config's
# paths minus paths-ignore (.github/codeql/codeql-config.yml, read below).
# A change to the CodeQL machinery itself scans every language (fail closed);
# those paths are full runs anyway, this keeps that true if a tier moves.
CODEQL_C_EXT = (".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".inc")
CODEQL_C_ROOTS = ("src/", "internal/", "vendored/")
CODEQL_EXT = {"javascript-typescript": (".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx",
                                        ".mts", ".cts", ".vue", ".html", ".htm"),
              "python": (".py",)}
CODEQL_FAIL_CLOSED_PREFIXES = (".github/workflows/", ".github/codeql/")
CODEQL_FAIL_CLOSED = {"Makefile.cbm", "scripts/build.sh", "scripts/ci/select-lanes.sh",
                      "scripts/ci/codeql-retrace-vendored.sh"}


def config_lists(path):
    """paths / paths-ignore of the CodeQL config: `key:` then `  - item` lines.
    None when the file is unreadable -- every interpreted scope then matches
    by extension alone (fail closed)."""
    try:
        lines = open(path, encoding="utf-8").read().splitlines()
    except OSError:
        return None
    lists, key = {"paths": [], "paths-ignore": []}, None
    for line in lines:
        if not line.strip() or line.lstrip().startswith("#"):
            continue
        m = re.match(r"^([A-Za-z-]+):\s*$", line)
        if m:
            key = m.group(1) if m.group(1) in lists else None
            continue
        m = re.match(r"^\s+-\s+['\"]?([^'\"#]+?)['\"]?\s*$", line)
        if m and key:
            lists[key].append(m.group(1).rstrip("/"))
        elif not line.startswith((" ", "\t")):
            key = None
    return lists


CODEQL_PATHS = config_lists(codeql_config)


def under(path, prefixes):
    return any(path == p or path.startswith(p + "/") for p in prefixes)


def codeql_fail_closed(path):
    return path in CODEQL_FAIL_CLOSED or path.startswith(CODEQL_FAIL_CLOSED_PREFIXES)


def codeql_scope(language, path):
    if codeql_fail_closed(path):
        return True
    if language == "c-cpp":
        return path.startswith(CODEQL_C_ROOTS) and path.endswith(CODEQL_C_EXT)
    if CODEQL_PATHS is not None and (not under(path, CODEQL_PATHS["paths"])
                                     or under(path, CODEQL_PATHS["paths-ignore"])):
        return False
    if language == "actions":
        return path.startswith(".github/workflows/") or \
            path.rsplit("/", 1)[-1] in ("action.yml", "action.yaml")
    return path.endswith(CODEQL_EXT[language])


def triggers(path, cat):
    found = []
    if path == "scripts/ci/select-lanes.sh":
        found.append("the lane selector itself")
    if path.startswith(".github/workflows/"):
        found.append("workflow")
    if path in HARNESS:
        found.append("test harness")
    if path == "Makefile.cbm":
        found.append("Makefile.cbm (registration and flag changes are indistinguishable by name)")
    if cat == "vendored":
        found.append("vendored code")
    if cat in ("other", "scripts-other") or not known(cat):
        found.append("path matched by no rule")
    return found


LIST_CAP = 3000   # the pulls/files API lists at most this many files


def truncated(paths, expect):
    """A list that may be missing files: at or over the API cap, or shorter
    than the PR's changed_files (a complete list is never shorter: every
    changed file is listed, renames add their old path on top)."""
    if len(paths) >= LIST_CAP:
        return f"{len(paths)} paths listed, at the {LIST_CAP}-file API cap"
    if expect is not None and expect >= LIST_CAP:
        return f"the PR changes {expect} files, over the {LIST_CAP}-file API cap"
    if expect is not None and len(paths) < expect:
        return f"{len(paths)} paths listed for {expect} changed files"
    return None


def select(paths, expect=None):
    paths = sorted({p for p in paths if p})
    by_cat, by_trigger = {}, {}
    for path in paths:
        cat = normalized(category(path))
        by_cat.setdefault(cat, []).append(path)
        for trigger in triggers(path, cat):
            by_trigger.setdefault(trigger, []).append(path)
    cats = set(by_cat)
    tier = tier_of(cats)
    lanes = R1[tier] | ALWAYS
    reasons = [f"category {c}: {len(p)} file(s), e.g. {p[0]}" for c, p in by_cat.items()]
    for when, extra in ADD_ONS:
        hit = sorted(cats & when)
        if hit and tier != "T0a-docs" and not extra <= lanes:
            reasons.append(f"add-on {'/'.join(hit)}: +{' +'.join(sorted(extra - lanes))}")
            lanes = lanes | extra
    if lanes & TEST_LEGS:
        lanes = lanes | {"shard-completeness"}
    for language, lane in sorted(CODEQL_LANES.items()):
        hits = [p for p in paths if codeql_scope(language, p)]
        if hits and lane not in lanes:
            more = f" (+{len(hits) - 1} more)" if len(hits) > 1 else ""
            reasons.append(f"codeql {language}: {hits[0]}{more}")
            lanes = lanes | {lane}
    for trigger, hits in by_trigger.items():
        more = f" (+{len(hits) - 1} more)" if len(hits) > 1 else ""
        reasons.append(f"full: {trigger}: {hits[0]}{more}")
    if not paths:
        reasons.append("full: empty file list (fail safe)")
    if tier == "T5-core":
        reasons.append("full: tier T5-core")
    short = truncated(paths, expect)
    if short:
        reasons.append(f"full: file_list_truncated: {short} (fail safe)")
    full = tier == "T5-core" or bool(by_trigger) or not paths or bool(short)
    if full:
        lanes = set(FULL)
    return {"full": full, "lanes": sorted(lanes), "reasons": sorted(reasons), "tier": tier}


def dump(obj):
    return json.dumps(obj, sort_keys=True, separators=(",", ":"))


def read_paths(path):
    with open(path, encoding="utf-8") as fh:   # universal newlines: CRLF == LF
        return [line.strip() for line in fh]


if mode == "list":
    print("\n".join(sorted(UNIVERSE)))
elif mode == "codeql-scope":
    scoped = sorted({p for p in read_paths(source) if p and codeql_scope(codeql_language, p)})
    if scoped:
        print("\n".join(scoped))
elif mode == "batch":
    with open(source, encoding="utf-8") as fh:
        for line in fh:
            if line.strip():
                row = json.loads(line)
                print(dump({"id": row["id"], **select(row["files"], row.get("expect_files"))}))
elif fmt == "github-output":
    decision = select(read_paths(source), expect)
    print(f"tier={decision['tier']}")
    print(f"full={'true' if decision['full'] else 'false'}")
    print(f"lanes={dump(decision['lanes'])}")
    print(f"reasons={dump(decision['reasons'])}")
else:
    print(dump(select(read_paths(source), expect)))
PY