Merge main to pick up shared CI contract fixes

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-10-22 00:34:24 +02:00
commit 8b2d840104
40 files changed
+1475 -406

No files matched your search

+27
View File
@@ -0,0 +1,27 @@
# CodeQL scope: the repository's first-party source.
#
# Read by .github/workflows/codeql.yml (every scan, full and PR) and by
# scripts/ci/select-lanes.sh (which changed paths select which language's PR
# scan). Keep the lists plain: one path prefix per line, no globs.
#
# paths / paths-ignore narrow what the INTERPRETED languages extract
# (javascript-typescript, python, actions). For c-cpp they have no effect:
# CodeQL extracts exactly what the traced build compiles. A PR's c-cpp scan
# keeps vendored C out by compiling it before tracing starts (except vendored
# files the PR changes); the full scan on main, the dry run and the release
# traces everything, vendored C included.
name: cbm first-party source
paths:
- src
- internal
- graph-ui
- pkg
- scripts
- tests
- .github
paths-ignore:
- internal/cbm/vendored
- graph-ui/vendor
- graph-ui/fixtures
- graph-ui/verification
- tests/fixtures
+1 -1
View File
@@ -1,5 +1,5 @@
# Reusable: lint only (cppcheck + clang-format).
# Security-static and CodeQL gate are separate — see _security.yml.
# Security-static is separate — see _security.yml; CodeQL — see codeql.yml.
name: Lint & Security
on:
+3 -94
View File
@@ -1,4 +1,6 @@
# Reusable: security-static + CodeQL gate.
# Reusable: security-static + license gate.
# CodeQL is not here: PRs do not run it. It runs on push to main, weekly, and
# inside dry-run.yml / release.yml, which call codeql.yml directly.
# Runs independently from lint/test/build — does not block the main pipeline.
# Affects overall workflow success status.
name: Security Gate
@@ -52,96 +54,3 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: scripts/audit-license-provenance.py
codeql-gate:
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'codeql-gate') }}
runs-on: ubuntu-latest
timeout-minutes: 240
# Without security-events:read this job inherits the workflow's
# contents:read, the code-scanning alert API answers 403, and the gate
# below reported "0 alerts" on every run it has ever made. actions:read is
# what the CodeQL-run poll above needs for the same reason.
permissions:
contents: read
security-events: read
actions: read
steps:
- name: Wait for CodeQL on current commit (max 150 min)
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# On pull_request events github.sha is the synthetic merge commit;
# CodeQL runs are recorded against the PR head SHA.
CURRENT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
echo "Waiting for CodeQL to complete on $CURRENT_SHA..."
# 300 attempts x 30s = 150 min. The previous budget was 90 x 30s = 45 min,
# which is shorter than CodeQL actually takes on this repository: a measured
# run on PR #1426 (head 7b72652a) completed with conclusion=success after
# 124 min, 7 minutes AFTER this gate had already given up. That marked at
# least five contributor PRs red for a scan that passed. The job's own
# timeout-minutes is 240, so 150 still leaves headroom.
for attempt in $(seq 1 300); do
LATEST=$(gh api "repos/${{ github.repository }}/actions/workflows/codeql.yml/runs?head_sha=$CURRENT_SHA&per_page=1" \
--jq '.workflow_runs[] | "\(.conclusion) \(.status)"' 2>/dev/null | head -1 || echo "")
if [ -z "$LATEST" ]; then
echo " $attempt/300: no run yet..."; sleep 30; continue
fi
CONCLUSION=$(echo "$LATEST" | cut -d' ' -f1)
STATUS=$(echo "$LATEST" | cut -d' ' -f2)
if [ "$STATUS" = "completed" ] && [ "$CONCLUSION" = "success" ]; then
echo "=== CodeQL passed ==="; exit 0
elif [ "$STATUS" = "completed" ]; then
echo "BLOCKED: CodeQL $CONCLUSION"; exit 1
fi
echo " $attempt/300: $STATUS..."; sleep 30
done
echo "BLOCKED: CodeQL timeout"; exit 1
- name: Check for open code scanning alerts
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`,
# which looks defensive and is the opposite. gh writes the API error
# BODY to stdout, so a 403 made ALERTS the string
# {"message":"Resource not accessible by integration",...,"status":"403"}0
# every `[` comparison then failed as non-integer, the `if` took its
# false branch, and the step printed "CodeQL gate passed (0 alerts)"
# and exited 0. That is exactly how a security gate should NEVER fail:
# silently, in the direction of "everything is fine". An unreadable
# alert list is now an error, not a zero.
# NOTE: this runs in a command substitution, so it must `return`, not
# `exit` -- an `exit` here would only end the subshell and let the
# caller continue with an empty count, recreating the very bug this
# replaces. The callers below check the status explicitly.
count_open_alerts() {
local body status
body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \
--jq 'length' 2>/dev/null)
status=$?
if [ $status -ne 0 ]; then
echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2
echo " Refusing to treat an unreadable alert list as zero." >&2
echo " Most likely cause: the job lacks security-events:read." >&2
return 1
fi
case "$body" in
''|*[!0-9]*)
echo "BLOCKED: alert count was not a number: ${body:-<empty>}" >&2
return 1
;;
esac
printf '%s' "$body"
}
echo "Waiting 60s for alert API to settle..."
sleep 60
ALERTS=$(count_open_alerts) || exit 1
sleep 15
ALERTS2=$(count_open_alerts) || exit 1
[ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2
if [ "$ALERTS" -gt 0 ]; then
echo "BLOCKED: $ALERTS open alert(s)"
gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \
--jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true
exit 1
fi
echo "=== CodeQL gate passed (0 alerts) ==="
+163 -16
View File
@@ -1,47 +1,194 @@
name: CodeQL SAST
# One analysis per language: c-cpp, javascript-typescript (graph-ui, pkg/npm),
# python, actions. Scope: .github/codeql/codeql-config.yml (first-party source).
#
# FULL scan -- push to main, weekly, dry run, release: every language; c-cpp
# traces the whole production build, vendored C included.
# PR scan -- pr.yml's `codeql` job (light: true), needed by ci-ok. One leg per
# language whose scope the PR changes (codeql-* lanes of
# scripts/ci/select-lanes.sh; an undeterminable diff or a change to the
# workflows, the build or the CodeQL machinery selects every language).
# Each leg analyses its language's WHOLE first-party scope, so cross-file
# data flow is kept; c-cpp compiles the vendored objects before tracing
# starts, except vendored files the PR changes. Diff-informed analysis then
# restricts the REPORTED alerts to the changed lines; what is extracted and
# analysed is never narrowed by the diff.
# There is no pull_request trigger and no job anywhere that polls for a CodeQL
# run: the former PR gate waited up to 150 min per PR for a full analysis and,
# with ~30 PRs pushed at once, starved PR CI for hours.
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
- cron: "23 5 * * 1"
workflow_dispatch:
inputs:
light:
description: "PR-style c-cpp build (vendored objects compiled untraced) -- to time it"
type: boolean
default: false
workflow_call:
inputs:
light:
description: "PR scan: vendored C compiled untraced except changed files; diff-informed results"
type: boolean
default: false
matrix:
description: 'The legs to run, {"include":[...]} (pr.yml: the selected codeql-* lanes); empty = every language'
type: string
default: ""
permissions:
contents: read
# One analysis per pull request and one per pushed branch. A newer push to main
# cancels the analysis of an older main commit; only the newest commit's
# results matter for main, and a merge-heavy day otherwise queues one 40-100
# minute analysis per merge ahead of every pull request run.
concurrency:
group: codeql-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
analyze:
runs-on: ubuntu-latest
# CodeQL needs security-events: write to upload results; scoped to this job so
# the workflow's top-level token stays read-only (Scorecard TokenPermissions).
strategy:
fail-fast: false
# The same legs (and keys) everywhere, so a PR's analysis and main's
# share one code-scanning configuration per language.
matrix: ${{ fromJSON(inputs.matrix || '{"include":[{"language":"c-cpp","build-mode":"manual","lane":"codeql-cpp"},{"language":"javascript-typescript","build-mode":"none","lane":"codeql-js"},{"language":"python","build-mode":"none","lane":"codeql-py"},{"language":"actions","build-mode":"none","lane":"codeql-actions"}]}') }}
# Superseded runs are always cancelled: a newer push to main cancels the
# analysis of an older main commit (batched merges -> only the newest main
# commit is scanned), a newer push to a PR cancels that PR's scan. Every
# other trigger gets a group of its own and is never cancelled: a dry run
# or a release tests ONE commit, and two dispatched together must not
# cancel each other's gate. Per language, so legs never cancel each other.
# In a called workflow `github` is the CALLER's context: event_name is
# pull_request for pr.yml, github.workflow names the dry run or release.
concurrency:
group: codeql-${{ matrix.language }}-${{ github.event_name == 'push' && github.ref || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('{0}-{1}', github.workflow, github.run_id) }}
cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
# CodeQL needs security-events: write to upload results (and read for the
# alert check); scoped to this job so the workflow's top-level token stays
# read-only (Scorecard TokenPermissions).
permissions:
security-events: write
actions: read
contents: read
env:
# One code-scanning configuration for every trigger. The action derives
# the key from the RUN's workflow file, which for a called workflow is
# the caller: pr.yml / dry-run.yml / release.yml would each open a
# configuration of their own, and on main an alert fixed by a later push
# would stay open in a stale one. The value is the key the push-to-main
# analysis has always used (the action documents this variable as
# settable for non-standard setups).
CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql.yml:analyze
# Diff-informed analysis: codeql-action v4.38.1 (CLI 2.27.0) reports a
# pull request's alerts only on its changed lines when the
# diff_informed_queries feature is on (default on; this variable forces
# it, src/feature-flags.ts). It applies only when the event payload is a
# pull request -- pr.yml's call -- and is inert on main, dry run, release.
CODEQL_ACTION_DIFF_INFORMED_QUERIES: ${{ inputs.light && 'true' || '' }}
# No compiler cache: a cache hit skips the compiler, and CodeQL extracts
# only what a traced compiler invocation actually compiles.
CBM_NO_CCACHE: "1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# The PR scan reads the merge commit's first parent (the base) to
# find the vendored files the PR changes.
fetch-depth: ${{ inputs.light && 2 || 1 }}
- name: Install build dependencies
if: ${{ matrix.language == 'c-cpp' }}
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev
# PR c-cpp: compile the third-party objects (tree-sitter grammars,
# sqlite, mimalloc, lz4, zstd, the tree-sitter runtime: ~1.3 GB of
# vendored C against ~45 MB first-party) BEFORE tracing starts, so the
# traced build below compiles first-party code only ...
- name: Build third-party objects (PR scan, untraced)
if: ${{ matrix.language == 'c-cpp' && inputs.light }}
run: scripts/build.sh --vendored-only
# ... except the vendored files this PR changes: those are rebuilt
# under tracing (an undeterminable diff leaves the list empty; the scan
# still covers all first-party C).
- name: Vendored files this PR changes (traced too)
if: ${{ matrix.language == 'c-cpp' && inputs.light }}
run: scripts/ci/codeql-retrace-vendored.sh "$RUNNER_TEMP/codeql-retrace.txt"
- name: Initialize CodeQL
uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
with:
languages: c-cpp
build-mode: manual
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql/codeql-config.yml
- name: Build for CodeQL analysis
- name: Build for CodeQL analysis (full)
if: ${{ matrix.language == 'c-cpp' && !inputs.light }}
run: scripts/build.sh
- name: Build for CodeQL analysis (PR scan, first-party + changed vendored)
if: ${{ matrix.language == 'c-cpp' && inputs.light }}
run: scripts/build.sh --reuse-vendored --retrace-from "$RUNNER_TEMP/codeql-retrace.txt"
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
with:
category: "/language:c-cpp"
category: "/language:${{ matrix.language }}"
# Alert policy: the default branch carries ZERO open code-scanning
# alerts. Any open alert on the analysed ref fails this leg: on main the
# commit turns red and GitHub notifies whoever triggered the run; the
# dry run and the release cannot pass; a PR (refs/pull/<n>/merge: what
# its changed lines introduce) fails ci-ok. With a zero baseline "any
# open alert" is exactly "a new alert"; a dismissal (with a reason, in
# the Security tab) holds on every ref. Every leg checks after its own
# upload, so each language's alerts are seen by at least its own leg.
# Moved from _security.yml's former PR-CI codeql-gate job.
- name: Check for open code scanning alerts
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ALERT_REF: ${{ github.ref }}
run: |
# FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`,
# which looks defensive and is the opposite. gh writes the API error
# BODY to stdout, so a 403 made ALERTS the string
# {"message":"Resource not accessible by integration",...,"status":"403"}0
# every `[` comparison then failed as non-integer, the `if` took its
# false branch, and the step printed "CodeQL gate passed (0 alerts)"
# and exited 0. That is exactly how a security gate should NEVER fail:
# silently, in the direction of "everything is fine". An unreadable
# alert list is now an error, not a zero.
# NOTE: this runs in a command substitution, so it must `return`, not
# `exit` -- an `exit` here would only end the subshell and let the
# caller continue with an empty count, recreating the very bug this
# replaces. The callers below check the status explicitly.
count_open_alerts() {
local body status
body=$(gh api "repos/${{ github.repository }}/code-scanning/alerts?state=open&ref=$ALERT_REF" \
--jq 'length' 2>/dev/null)
status=$?
if [ $status -ne 0 ]; then
echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2
echo " Refusing to treat an unreadable alert list as zero." >&2
echo " Most likely cause: the job lacks security-events:read." >&2
return 1
fi
case "$body" in
''|*[!0-9]*)
echo "BLOCKED: alert count was not a number: ${body:-<empty>}" >&2
return 1
;;
esac
printf '%s' "$body"
}
echo "Waiting 60s for alert API to settle..."
sleep 60
ALERTS=$(count_open_alerts) || exit 1
sleep 15
ALERTS2=$(count_open_alerts) || exit 1
[ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2
if [ "$ALERTS" -gt 0 ]; then
echo "::error title=Open code scanning alerts::$ALERTS open alert(s) on $ALERT_REF -- fix, or dismiss with a reason in the Security tab"
echo "BLOCKED: $ALERTS open alert(s)"
gh api "repos/${{ github.repository }}/code-scanning/alerts?state=open&ref=$ALERT_REF" \
--jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true
exit 1
fi
echo "=== CodeQL gate passed (0 alerts) ==="
+17 -7
View File
@@ -2,9 +2,10 @@
# Each step can be skipped for faster iteration.
#
# Pipeline: lint → test → build + candidate scan/select → smoke/soak
# Security: security-static + codeql-gate (independent island)
# Security: security-static + license-gate (independent island)
# CodeQL: codeql.yml full scan of this commit + alert check (independent island)
#
# Security does NOT block lint/test/build/smoke/soak. All jobs must pass
# Security and CodeQL do NOT block lint/test/build/smoke/soak. All jobs must pass
# for the overall workflow to be green.
#
# Candidate VirusTotal scanning and tuple-local selection are part of the
@@ -62,15 +63,24 @@ jobs:
# ── Security (independent island — does not block main pipeline) ──
security:
uses: ./.github/workflows/_security.yml
# The called job requests security-events:read to count code-scanning
# alerts; a reusable workflow cannot request more than its caller grants,
# so withholding this here fails the whole run at startup.
# No job in _security.yml reads code scanning or the Actions API any more
# (the CodeQL gate moved to codeql.yml), so the token stays read-only.
permissions:
contents: read
security-events: read
actions: read
secrets: inherit
# ── CodeQL: full scan of this exact commit + the open-alert check ──
# Every language, vendored C included (PRs run the narrower PR scan).
# codeql.yml uploads results and reads the alerts: a called workflow
# cannot request more than its caller grants, so withholding either
# permission fails the whole run at startup.
codeql:
uses: ./.github/workflows/codeql.yml
permissions:
contents: read
security-events: write
actions: read
# ── Lint (cppcheck + clang-format) ────────────────────────────
lint:
if: ${{ inputs.skip_lint != true }}
+29 -6
View File
@@ -26,13 +26,10 @@ jobs:
security:
needs: [changes]
uses: ./.github/workflows/_security.yml
# The called job requests security-events:read to count code-scanning
# alerts; a reusable workflow cannot request more than its caller grants,
# so withholding this here fails the whole run at startup.
# No job in _security.yml reads code scanning or the Actions API any more
# (the CodeQL gate moved to codeql.yml), so the token stays read-only.
permissions:
contents: read
security-events: read
actions: read
secrets: inherit
with:
lanes: ${{ needs.changes.outputs.lanes }}
@@ -128,6 +125,7 @@ jobs:
full: ${{ steps.select.outputs.full }}
lanes: ${{ steps.select.outputs.lanes }}
smoke: ${{ steps.select.outputs.smoke }}
codeql: ${{ steps.select.outputs.codeql }}
steps:
- name: List the changed files
env:
@@ -180,6 +178,10 @@ jobs:
SMOKE='[{"os":"ubuntu-latest","lane":"smoke-ubuntu"},{"os":"macos-14","lane":"smoke-mac"},{"os":"windows-latest","lane":"smoke-win"}]'
LANES=$(sed -n 's/^lanes=//p' "$RUNNER_TEMP/selection.txt")
echo "smoke={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$SMOKE")}" >> "$GITHUB_OUTPUT"
# The CodeQL legs, tagged the same way (codeql.yml's full matrix has
# the same entries, so PR and main share a configuration).
CODEQL='[{"language":"c-cpp","build-mode":"manual","lane":"codeql-cpp"},{"language":"javascript-typescript","build-mode":"none","lane":"codeql-js"},{"language":"python","build-mode":"none","lane":"codeql-py"},{"language":"actions","build-mode":"none","lane":"codeql-actions"}]'
echo "codeql={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$CODEQL")}" >> "$GITHUB_OUTPUT"
# ── Light smoke: build the PRODUCTION binary and run the core smoke on the
# three RELIABLE NATIVE platforms. Catches built-binary regressions at PR
@@ -361,13 +363,34 @@ jobs:
- name: Contract steps only
run: scripts/test.sh --contracts-only
# ── CodeQL PR scan (lanes codeql-cpp / -js / -py / -actions): one leg per
# language whose first-party scope this PR changes; each analyses that
# language's whole first-party scope, c-cpp with the vendored objects
# compiled before tracing (except vendored files the PR changes), and
# reports alerts on the changed lines (diff-informed). A job of this run,
# needed by ci-ok -- nothing waits for a separate CodeQL run. Push to
# main, dry run and release run the full scan (codeql.yml). ──
codeql:
needs: [changes]
if: ${{ !cancelled() && needs.changes.result == 'success' && (contains(fromJSON(needs.changes.outputs.lanes), 'codeql-cpp') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-js') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-py') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-actions')) }}
uses: ./.github/workflows/codeql.yml
# codeql.yml uploads results and reads alerts: a called workflow cannot
# request more than its caller grants (startup failure otherwise).
permissions:
contents: read
security-events: write
actions: read
with:
light: true
matrix: ${{ needs.changes.outputs.codeql }}
ci-ok:
# The one required context (besides dco) — fails unless every PR stage
# succeeded, so matrix renames can never silently deadlock merges, and
# every lane the selector chose ran green: `skipped` is OK only for a lane
# nobody selected (a selected lane that skipped is a silent gate loss).
# memwaste and the test-impact shadow are report-only and not needed here.
needs: [changes, security, lint, test, pr-smoke, contracts]
needs: [changes, security, lint, test, pr-smoke, contracts, codeql]
if: ${{ always() }}
runs-on: ubuntu-latest
timeout-minutes: 5
+20 -10
View File
@@ -1,7 +1,8 @@
# Release pipeline: lint → test → build → smoke/soak → draft → verify → publish
#
# Security (security-static + codeql-gate) starts immediately and runs in
# parallel with lint/test/build/smoke/soak. It only blocks the final verify
# Security (security-static + license-gate) and CodeQL (codeql.yml: full scan
# of this commit + the open-alert check) start immediately and run in
# parallel with lint/test/build/smoke/soak. They only block the final verify
# step — everything else proceeds independently.
name: Release
@@ -44,15 +45,24 @@ jobs:
# ── Security (starts immediately, blocks only verify) ───────────
security:
uses: ./.github/workflows/_security.yml
# The called job requests security-events:read to count code-scanning
# alerts; a reusable workflow cannot request more than its caller grants,
# so withholding this here fails the whole run at startup.
# No job in _security.yml reads code scanning or the Actions API any more
# (the CodeQL gate moved to codeql.yml), so the token stays read-only.
permissions:
contents: read
security-events: read
actions: read
secrets: inherit
# ── CodeQL (starts immediately, blocks only verify) ──────────────
# Full scan of this exact commit (every language, vendored C included) +
# the open-alert check. codeql.yml uploads results and reads the alerts:
# a called workflow cannot request more than its caller grants, so
# withholding either permission fails the whole run at startup.
codeql:
uses: ./.github/workflows/codeql.yml
permissions:
contents: read
security-events: write
actions: read
# ── 0. Preflight: refuse malformed dispatch inputs ──────────────
# The tag is inputs.version VERBATIM. A bare "0.10.7" publishes a release the
# installers can never resolve (they fetch releases/download/v<version>/...),
@@ -312,12 +322,12 @@ jobs:
body: ${{ inputs.release_notes || '' }}
generate_release_notes: ${{ inputs.release_notes == '' }}
# ── 7. Verify + Publish (requires security gate) ───────────────
# ── 7. Verify + Publish (requires the security and CodeQL gates) ─
verify:
needs: [release-draft, security]
needs: [release-draft, security, codeql]
# Explicit condition: a deliberately skipped test ancestor must not skip
# the publish chain; the draft itself proves smoke and soak both succeeded.
if: ${{ !cancelled() && !failure() && needs.security.result == 'success' && needs.release-draft.result == 'success' }}
if: ${{ !cancelled() && !failure() && needs.security.result == 'success' && needs.codeql.result == 'success' && needs.release-draft.result == 'success' }}
runs-on: ubuntu-latest
permissions:
contents: write
+1 -1
View File
@@ -52,7 +52,7 @@ Operational authority is stricter than code review authority.
| Operation | Current authority | Project rule |
| --- | --- | --- |
| PR validation (`pr.yml`, DCO, CodeQL) | Automatic | Anyone may trigger it by opening or updating a PR. Required checks must pass. |
| PR validation (`pr.yml`, DCO, PR CodeQL scan) | Automatic | Anyone may trigger it by opening or updating a PR. Required checks must pass. |
| Dry run (`dry-run.yml`) | `@DeusData` | Delegated release operators may run dry runs after promotion. Dry-run delegation does not imply release authority. |
| Smoke/soak/repro manual runs | `@DeusData` | Area reviewers may operate these runs when delegated for diagnosis. Results are advisory. |
| Release workflow (`release.yml`) | `@DeusData` only | Owner-only until a release operator is explicitly promoted. Publishing, replacing releases, and tag movement remain owner-gated. |
+10
View File
@@ -1299,6 +1299,16 @@ $(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c $(BUILD_DIR)/.build-co
OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)
# The part of OBJS_VENDORED_PROD that holds no first-party code: everything but
# prod_lsp_all.o (first-party LSP unity) and prod_preprocessor.o (first-party
# code around vendored simplecpp). CodeQL's light PR scan builds these BEFORE
# it starts tracing (scripts/build.sh --vendored-only), so the traced build
# (--reuse-vendored) compiles first-party code only: the grammars alone are
# most of the 1.3 GB of vendored C, against ~45 MB of first-party code.
OBJS_THIRD_PARTY_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)
.PHONY: cbm-third-party
cbm-third-party: $(OBJS_THIRD_PARTY_PROD)
MAIN_SRC = src/main.c
# Rebuild when the build CONFIGURATION changes, not only when sources do. The
+34 -29
View File
@@ -1,9 +1,9 @@
# codebase-memory-mcp: local code intelligence for developers and AI agents
# codebase-memory-mcp: local code intelligence for AI agents via MCP
[![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest)
[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
[![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/release.yml?branch=main&label=main%20CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/release.yml)
[![C tests](https://img.shields.io/badge/C_tests-9388_cases-blue)](#tests)
[![Native and frontend tests](https://img.shields.io/badge/tests-native_%2B_frontend-blue)](#tests)
[![MCP tools](https://img.shields.io/badge/MCP_tools-17-blue)](#mcp-tools)
[![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp)
[![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-type_resolution-blue)](#hybrid-lsp)
@@ -14,27 +14,28 @@
[![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev)
[![VirusTotal](https://img.shields.io/badge/VirusTotal-scanned_every_release-brightgreen?logo=virustotal)](https://github.com/DeusData/codebase-memory-mcp/releases/latest)
[![arXiv](https://img.shields.io/badge/arXiv-2603.27277-b31b1b?logo=arxiv)](https://arxiv.org/abs/2603.27277)
[![Join the community on Discord](https://img.shields.io/badge/Discord-Join_the_community-5865F2?logo=discord&logoColor=white)](https://discord.gg/KguGXxg2X)
**Understand your repository, from source code to the systems around it.** codebase-memory-mcp connects code, documentation, configuration, and infrastructure in a persistent local knowledge graph. Use it to find an unfamiliar feature, follow a call across services, trace a deployment back to its source, or assess which parts of a project a change could affect.
**Understand your repository, from source code to the systems around it.** codebase-memory-mcp is an MCP server that connects code, documentation, configuration, and infrastructure in a persistent local knowledge graph. Use it to find an unfamiliar feature, follow a call across services, trace a deployment back to its source, or assess which parts of a project a change could affect.
Give your coding agent structural context through [Model Context Protocol](https://modelcontextprotocol.io/) (MCP), query the graph from the CLI, or explore it yourself in **CBM Lens Beta**. Lens brings together an IDE-style source reader, architecture maps, the Galaxy graph, coverage diagnostics, architecture decisions, and optional local code chat.
Give your coding agent structural context through [Model Context Protocol](https://modelcontextprotocol.io/) (MCP), or query the graph directly from the CLI. **The MCP server is the core of the project.** Its optional visual companion, **CBM Lens, is in open beta (v0.0.1)**, with a source reader, architecture maps, the Galaxy graph, coverage diagnostics, architecture decisions, and local code chat.
The native C backend exposes **17 MCP tools**, parses [languages and file formats](#language-support) with vendored grammars, and adds [Hybrid LSP](#hybrid-lsp) type resolution for supported languages. The installer supports **45 automatic or conditional client integrations**, including client configuration, skills, agent profiles, and applicable hooks. Indexing and graph queries need no hosted service or API key.
The native C backend exposes **17 MCP tools**, parses [languages and file formats](#language-support) with vendored grammars, and adds [Hybrid LSP](#hybrid-lsp) type resolution for supported languages. The installer supports **45 automatic or conditional client integrations**, including client configuration, skills, agent profiles, and applicable hooks. Indexing and graph queries need no hosted service or API key. Focused graph queries can reduce repeated file reads and token use; see the [measured results](#performance).
[Quick start](#quick-start) · [Capabilities](#features) · [Documentation](#documentation-connected-to-code) · [Infrastructure](#infrastructure-and-deployment) · [Smart CI](#smart-ci-and-test-impact) · [Explore CBM Lens](#cbm-lens-beta-code-visualization-and-source-navigation) · [Agent setup](#agent-setup-skills-profiles-and-hooks) · [MCP tools](#mcp-tools)
> **Local processing:** Source analysis and graph storage run on your machine. CBM Lens can optionally download a model for browser AI after you enable it; inference stays on the device. The installer writes agent configuration for the clients you select. See [Security](#security) for release verification and [SECURITY.md](SECURITY.md) for reporting issues.
**Architecture overview.** Folders become nested platforms, files and symbols become selectable blocks, and arrows show indexed relationships. The recording zooms into the map of this repository, selects `src/pipeline`, opens the area, selects `artifact.c` and opens its source. Height can show source size; hotspot gravity reflects static fan-in. [Still image](docs/images/lens-architecture.png).
**See the graph with CBM Lens (open beta v0.0.1).** Folders become nested platforms, files and symbols become selectable blocks, and arrows show indexed relationships. The recording opens `src/foundation` in the map of this repository, selects `arena.c` and opens its source. Height can show source size; hotspot gravity reflects static fan-in. [Still image](docs/images/lens-architecture.png).
<p align="center">
<img src="docs/images/lens-overview.gif" alt="CBM Lens Beta architecture walkthrough: zoom into the source folders of this repository, open src/pipeline, select artifact.c and open its source" width="1000">
<img src="docs/images/lens-overview.gif" alt="CBM Lens Beta architecture walkthrough: open src/foundation, select arena.c and read its source beside the map" width="1000">
</p>
**Galaxy.** Explore the code knowledge graph in 3D. The recording searches `cbm_pipeline_run`, follows its outgoing calls one layer further, switches to the hierarchy and shows the path to `cbm_store_check_integrity`. Edge pulses show relationship direction; they do not represent live execution. [Current still: two dependency layers around `cbm_max_file_bytes`](docs/images/lens-galaxy.jpg).
**Galaxy.** Explore the code knowledge graph in 3D. The recording follows the outgoing relationships of `cbm_arena_alloc` one layer further, switches to the hierarchy and shows the two-hop call path to `cbm_alloc`. Edge pulses show relationship direction; they do not represent live execution. [Still: two dependency layers around `cbm_max_file_bytes`](docs/images/lens-galaxy.jpg).
<p align="center">
<img src="docs/images/lens-galaxy.gif" alt="Code knowledge graph in CBM Lens Beta Galaxy: search a function, expand its outgoing calls, switch to the hierarchy and follow a path" width="1000">
<img src="docs/images/lens-galaxy.gif" alt="Code knowledge graph in CBM Lens Beta Galaxy: expand cbm_arena_alloc, switch to the hierarchy and follow its call path through arena_grow to cbm_alloc" width="1000">
</p>
**Explore.** Browse files in an IDE-style workspace, read syntax-highlighted source in Monaco, and keep the code graph alongside it. File impact shows what depends on the open file, and Code details shows the callers and calls of the symbol under the caret. Select code to [ask the local chat about it](#chat-with-your-codebase-locally).
@@ -52,11 +53,11 @@ The native C backend exposes **17 MCP tools**, parses [languages and file format
</details>
These captures show CBM Lens Beta on 12–13 October 2026, using public source from this repository. See [capture details](docs/images/README.md) for each capture's build, indexed scope and recording setup.
The high-resolution GIFs show CBM Lens Beta on 20 October 2026; the still images were captured on 12–13 October. All show public source from this repository. See [capture details](docs/images/README.md) for each capture's build, indexed scope and recording setup.
## Why codebase-memory-mcp
- **Find the right code with context**: combine symbol, text, and semantic search with callers, dependencies, source snippets, and file outlines. Reduce repeated file exploration; see [Performance](#performance) for scoped measurements.
- **Find the right code with context**: combine symbol, text, and semantic search with callers, dependencies, source snippets, and file outlines. Targeted queries and compact, paged results can reduce token use and repeated file reads; see [Performance](#performance) for scoped measurements.
- **Connect documentation to implementation**: index document sections, PDF text pages, architecture decisions, and supported references into code. Find the explanation alongside the symbol.
- **Follow the system beyond source files**: trace supported relationships through Docker, Kubernetes, Helm, Kustomize, Terraform/OpenTofu, Terragrunt, Ansible, scripts, Make, and CI configuration.
- **Understand a change before running it**: inspect affected symbols, compare indexed snapshots, and find related tests. [Experimental Smart CI](#smart-ci-and-test-impact) combines static relationships and per-test coverage for supported C/C++ test harnesses, falling back to broader runs when evidence is insufficient.
@@ -160,15 +161,15 @@ Thin frontends still write immediate startup and session-specific errors to thei
All active CBM processes must run the exact same version, executable build, coordination ABI, and canonical cache root. Equivalent `CBM_CACHE_DIR` aliases resolve to the same root; a genuinely different root is rejected while any CBM process is active. MCP servers, hooks, one-shot CLI commands, temporary index workers, and the daemon share a crash-safe OS admission barrier; starting an ordinary conflicting process fails before doing work and records an explicit conflict in `${CBM_CACHE_DIR}/logs/daemon-conflicts.ndjson`.
The native `install`, `update`, and `uninstall` commands are the deliberate exception to that conflict rule. Download, verification, and private same-filesystem staging happen first so a bad candidate never disrupts active work. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build.
The native `install` and `uninstall` commands are the deliberate exception to that conflict rule. For installation, download, verification, and private same-filesystem staging finish before activation so a bad candidate never disrupts active work. The native `update` command prints the installer command that performs the update. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build.
Package-manager setup (npm, PyPI, or Go) verifies and caches the native executable for that package. It does not replace the active native installation and therefore does not stop running CBM sessions. When that cached binary is executed, it still enters the same exact-build admission barrier. The shell and PowerShell installers invoke the verified candidate's native `install` command, so they do receive the full account-wide activation guarantee.
The ordinary `cli` mode runs one command locally, but it is not disconnected from the daemon above: it connects to the shared coordination daemon (starting one if none is already running) for that same admission barrier plus per-project locks on graph mutations. Its connection is a `cli_session`: it never registers with the background watcher and never starts a UI, and it holds the admission lease only for the command's lifetime. If the command's own connection is what started the daemon, that daemon exits again once the command closes and no other session is attached; if a daemon was already running, the command simply joins and leaves it exactly as found. While the command is running, a temporary monitor separately lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details.
The ordinary `cli` mode runs one command locally, but it is not disconnected from the daemon above: it connects to the shared coordination daemon (starting one if none is already running) for that same admission barrier plus per-project locks on graph mutations. Its connection is a `cli_session`: it does not register a project watcher or open a browser, and it holds the admission lease only for the command's lifetime. The shared daemon may serve Lens according to its UI configuration and available assets. If the command's own connection is what started the daemon, that daemon exits again once the command closes and no other session is attached; if a daemon was already running, the command simply joins and leaves it exactly as found. While the command is running, a temporary monitor separately lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details.
### CBM Lens Beta: code visualization and source navigation
CBM Lens Beta is the browser interface included in standard native releases and UI-enabled source builds. It builds on [CodeAtlasWeb](graph-ui/README.md) by [Bernhard Jackiewicz](https://github.com/BernhardJackiewicz) (design and tracking in #1964). Lens is served on loopback. Browser AI downloads its optional model only after you enable it; inference stays on-device. Each panel names where its facts come from, and what the index did not record is shown as a gap. Start the local server:
**CBM Lens is an optional visual companion to the MCP server, currently in open beta (v0.0.1).** Standard native releases include its browser interface, so you can launch Lens directly after installation; UI-enabled source builds include it too. It builds on [CodeAtlasWeb](graph-ui/README.md) by [Bernhard Jackiewicz](https://github.com/BernhardJackiewicz) (design and tracking in #1964). Lens is served on loopback. Browser AI downloads its optional model only after you enable it; inference stays on-device. Each panel names where its facts come from, and what the index did not record is shown as a gap. Start the local server:
```bash
codebase-memory-mcp daemon start --open --port=9749
@@ -215,9 +216,9 @@ Open **Chat** with the panel button at the top right to ask about the code you a
Questions about callers and callees, such as "Who calls cbm_pipeline_run?", are answered from the loaded graph without the model, and the answer says so. When you select a symbol, the chat lists its indexed facts and the lines of its code, and the model adds one sentence; that sentence is left out when it claims something the source does not show. Text written by the model is labelled as such.
The recording shows a question about callers answered from the indexed graph, next to the automatic explanation of the selected symbol. [Still image](docs/images/lens-chat.png).
The recording asks who calls `arena_grow` and what it calls, with both answers listed directly from the indexed graph beside the selected symbol. [Still image of a model explanation and graph answer](docs/images/lens-chat.png).
<p align="center"><img src="docs/images/lens-chat.gif" alt="Chat with your codebase locally in CBM Lens Beta: a selected symbol, its listed facts and a caller question answered from the indexed graph" width="1000"></p>
<p align="center"><img src="docs/images/lens-chat.gif" alt="Chat with your codebase locally in CBM Lens Beta: callers and callees of arena_grow answered directly from the indexed graph" width="1000"></p>
The chat history is kept for 24 hours in this browser, per project. Each new message uses the source currently open in the reader; earlier source snapshots are not silently accumulated, and messages about another file or selection are not sent along. If a prompt is too long for the model, older messages are left out first; if it still does not fit, CBM Lens asks you to select less code instead of cutting the code. This chat explains the context you give it; it cannot edit files or run tools. Check its answers against the source. The small default model words things loosely, especially in languages other than English.
@@ -238,7 +239,7 @@ The Agents workspace and activity overlays are hidden in normal builds. To inclu
VITE_EXPERIMENTAL_AGENTS=true scripts/build.sh --with-ui
```
Activity comes from explicitly configured tool hooks. The bundled adapter supports Claude Code PostToolUse; other clients need their own adapter. Recorded events go to the local daemon, and the reader must enable activity before the browser polls them. This does not automatically observe every agent. See the [frontend setup guide](graph-ui/README.md#die-agentenebene-was-live-arbeitende-agenten-auf-dem-graphen-zeigen).
Activity comes from explicitly configured tool hooks. The bundled adapter supports Claude Code PostToolUse; other clients need their own adapter. Recorded events go to the local daemon, and the reader must enable activity before the browser polls them. This does not automatically observe every agent. See the [frontend setup guide](graph-ui/README.md#experimental-agent-activity).
</details>
@@ -273,7 +274,7 @@ bash "<install-dir>/install.sh"
```powershell
# Windows
powershell -ExecutionPolicy Bypass -File "<install-dir>\install.ps1"
powershell -File "<install-dir>\install.ps1"
```
The install script is placed next to the binary at install time, so the printed path resolves beside the executable. It is idempotent, so re-running it *is* the update: it stops the daemon, retires the running binary, installs the new one, and cleans up.
@@ -467,15 +468,15 @@ Published backend benchmarks on Apple M3 Pro. These are separate from the CBM Le
| Dead code detection | ~150ms | Full graph scan with degree filtering |
| Trace call path (depth=5) | <10ms | BFS traversal |
**RAM-first pipeline**: All indexing runs in memory (LZ4 HC compressed read, in-memory SQLite, single dump at end). Memory is released back to the OS after indexing completes.
**Memory-aware pipeline**: Indexing keeps hot data in memory and can spill extraction results to disk when needed. Memory is released back to the OS after indexing completes.
**Token efficiency, within the reported workloads**: The project's [original README report](https://github.com/DeusData/codebase-memory-mcp/blob/4ff6feeab639c4668da0f491930326ed5ec63093/README.md#performance) lists ~3,400 tokens for five structural queries via codebase-memory-mcp versus ~412,000 via file-by-file exploration: approximately **120× fewer tokens** for that comparison. The separate [31-repository preprint](https://arxiv.org/html/2603.27277v1#S4.SS1) reports **10× lower token use** and **2.1× fewer tool calls**, with answer-quality scores of **83% for CBM versus 92% for file-by-file exploration**. These are scoped backend measurements, not a promised saving for every task or a benchmark of CBM Lens. Actual savings depend on the question, repository, agent, and required verification. The study's [system-performance measurements](https://arxiv.org/html/2603.27277v1#S4.SS3) include the Linux indexing and Cypher timings above.
**More focused context, fewer tokens**: The project's [31-repository preprint](https://arxiv.org/html/2603.27277v1#S4.SS1) reports **10× lower token use** and **2.1× fewer tool calls** compared with file-by-file exploration. These March 2026 backend measurements cover the study's workloads and agent setup; the paper documents the full evaluation. An earlier [five-query comparison](https://github.com/DeusData/codebase-memory-mcp/blob/4ff6feeab639c4668da0f491930326ed5ec63093/README.md#performance) reported ~3,400 versus ~412,000 tokens for structural queries. The study's [system-performance measurements](https://arxiv.org/html/2603.27277v1#S4.SS3) include the Linux indexing and Cypher timings above.
To measure comparable quality, latency, and agent-efficiency metrics on your own workload, see [Measuring quality, latency, and agent savings](docs/MEASURING_SAVINGS.md). Exact reproduction of the figures above requires the original inputs and raw artifacts.
Explore the [measurement guide](docs/MEASURING_SAVINGS.md) to evaluate quality, latency and token use on your own workload.
## Troubleshooting & Diagnostics
codebase-memory-mcp runs **100% locally and collects no telemetry**: your code, queries, environment, and usage never leave your machine. That privacy guarantee also means that when you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no data at all unless you choose to send it.** Here is how to capture it yourself.
codebase-memory-mcp **indexes and queries locally and collects no telemetry**. When you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no diagnostic data unless you choose to send it.** Here is how to capture it yourself.
### Capture a diagnostics log
@@ -614,7 +615,7 @@ Counts on `main` at `4d9592b5`, where CBM Lens arrived. Native and Go counts use
| npm package wrapper | 24 tests | Counted in the source |
| Python package wrapper | 43 tests | Counted in the source |
Run the test suite (9,388 tests across 189 suites):
Run the native test suite:
```bash
scripts/test.sh # full: clean sanitizer build + all suites + guards
@@ -864,7 +865,7 @@ codebase-memory-mcp cli search_graph --project my-project --label Function --for
codebase-memory-mcp cli list_projects --format json --detail stats | jq '.projects[].name'
```
JSON arguments can also be piped on stdin, for tools that take arguments. A tool whose input schema declares none; `list_projects`; never reads stdin, so it stays responsive when it inherits a pipe the caller never closes (the default for `child_process.spawn` and similar wrappers). Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin.
Tools requiring arguments accept JSON piped on stdin. Tools with only optional arguments, such as `list_projects`, accept flags, `--args-file`, or redirected JSON files without waiting on an inherited pipe. Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin.
## MCP Tools
@@ -873,7 +874,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool
| Tool | Description |
|------|-------------|
| `index_repository` | Index a repository into the graph. Auto-sync keeps it fresh only while it is the project of an open MCP session; repos indexed from the CLI or from another project's session need a re-run (`index_status` shows `watch`). Waits for the whole index by default; pass `async: true` to start it in the daemon and return at once, then poll with `status: true` (see below). |
| `list_projects` | List all indexed projects with node/edge counts. |
| `list_projects` | List indexed projects with paging; use `detail: "stats"` for node/edge counts and database sizes. |
| `delete_project` | Remove a project and all its graph data. |
| `index_status` | Check indexing status of a project. |
| `check_index_coverage` | Check whether exact paths or a scope are indexed and fresh. A clean result means no recorded gap, not proof of completeness. |
@@ -989,7 +990,7 @@ is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
| `CBM_DOWNLOAD_URL` | *(GitHub releases)* | Override the download URL for updates. Used for testing or self-hosted deployments. |
| `CBM_LOG_LEVEL` | role-aware | Set the minimum log level. Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and its supervised index workers default to `info` so lifecycle and liveness records remain available. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none`; or their numeric equivalents `0` to `4`. A physical worker retains INFO liveness records even under a stricter override because its private log drives the supervisor's no-progress timeout. Frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. |
| `CBM_WORKERS` | *(detected)* | Override the parallel-indexing worker count returned by `cbm_default_worker_count`. Useful inside containers where `sysconf(_SC_NPROCESSORS_ONLN)` reports host CPUs rather than the cgroup's effective quota. Range 1 to 256; invalid values are ignored with a warning. |
| `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the `ram_fraction × total_RAM` default. Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). |
| `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the derived default (`ram_fraction × total_RAM`, adjusted for available memory and daemon worker caps). Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). |
| `CBM_DUMP_VERIFY_MIN_RATIO` | `0.5` | After indexing, compare persisted SQLite node count to the in-memory dump count. When persisted nodes fall below this fraction of committed nodes (and committed > 50), `index_repository` returns `status:"degraded"` instead of silent `indexed`. Range 0 to 1; set `0` to disable. Invalid values are ignored with a warning. |
Environment used by daemon-owned components (such as diagnostics, daemon logging, and process-wide indexing resource limits) is captured from the first daemon-backed session that starts the daemon. Later sessions join that process and cannot replace those values. To change them, close all daemon-backed sessions, update the relevant agent configurations consistently, and restart a session. `CBM_ALLOWED_ROOT` remains session-specific, a conflicting `CBM_CACHE_DIR` is rejected, and a one-shot CLI command is not exempt from the rule above: it connects to the coordination daemon like any other session, starting one if none is running. Its own environment becomes the captured daemon-owned environment only when its invocation is the one that starts the daemon; joining an already-running daemon, it inherits that daemon's already-captured values instead.
@@ -1019,9 +1020,9 @@ Need the full config-file reference? See [docs/CONFIGURATION.md](docs/CONFIGURAT
Each entry maps an extension (which **must** start with `.`) to a language name. Language names are matched **case-insensitively**. Accepted values (aliases in parentheses) are:
`bash` (`sh`), `c`, `c++` (`cpp`), `c#` (`csharp`), `clojure`, `cmake`, `cobol`, `common lisp` (`commonlisp`, `lisp`), `css`, `cuda`, `dart`, `dockerfile`, `elixir`, `elm`, `emacs lisp` (`emacslisp`), `erlang`, `f#` (`fsharp`), `form`, `fortran`, `glsl`, `go`, `graphql`, `groovy`, `haskell`, `hcl` (`terraform`), `html`, `ini`, `java`, `javascript`, `json`, `julia`, `kotlin`, `lean`, `lua`, `magma`, `makefile`, `markdown`, `matlab`, `meson`, `nix`, `objective-c` (`objc`), `ocaml`, `perl`, `php`, `protobuf`, `python`, `r`, `ruby`, `rust`, `scala`, `scss`, `sql`, `svelte`, `swift`, `toml`, `tsx`, `typescript`, `verilog`, `vimscript`, `vue`, `wolfram`, `xml`, `yaml`, `zig`.
`arkts`, `asciidoc`, `bash` (`sh`), `c`, `c++` (`cpp`), `c#` (`csharp`), `chialisp`, `clojure`, `cmake`, `cobol`, `common lisp` (`commonlisp`, `lisp`), `css`, `cuda`, `dart`, `diagram`, `dockerfile`, `elixir`, `elm`, `emacs lisp` (`emacslisp`), `erlang`, `f#` (`fsharp`), `form`, `fortran`, `glsl`, `go`, `graphql`, `groovy`, `haskell`, `hcl` (`terraform`), `html`, `ini`, `java`, `javascript`, `jinja2` (`jinja`), `json`, `julia`, `kotlin`, `lean`, `lua`, `magma`, `makefile`, `markdown`, `matlab`, `meson`, `mojo`, `nix`, `objective-c` (`objc`), `ocaml`, `openapi`, `pdf`, `perl`, `php`, `plsql`, `protobuf`, `python`, `r`, `ruby`, `rust`, `scala`, `scss`, `sql`, `svelte`, `swift`, `toml`, `tsx`, `typescript`, `verilog`, `vimscript`, `vue`, `wolfram`, `xml`, `yaml`, `zig`.
Project config overrides global for conflicting extensions. An entry whose language name is unknown, or whose extension does not start with `.`, is skipped and a warning is logged to stderr (shown at the default `info` log level). Missing config files are ignored.
Project config overrides global for conflicting extensions. An entry whose language name is unknown, or whose extension does not start with `.`, is skipped and logged as a warning. Missing config files are ignored.
## Persistence
@@ -1111,7 +1112,7 @@ Every release is verified through a multi-layer pipeline before publication:
- **Sigstore cosign**: keyless signatures on all artifacts; bundles included in every release
- **SHA-256 checksums**: `checksums.txt` published with every release; verified by both install scripts before extraction
- **CodeQL SAST**: blocks release pipeline if any open alerts remain
- **No language-runtime dependency chain**: the native backend uses libraries vendored at compile time, and the frontend is embedded by the UI build
- **No language-runtime dependency chain**: the native backend needs no language-runtime environment; UI builds embed the frontend
### VirusTotal release scans
@@ -1130,6 +1131,10 @@ The [v0.11.0 release notes](https://github.com/DeusData/codebase-memory-mcp/rele
The release also publishes [candidate scan results](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/virustotal-candidate-results.tsv), [binary provenance](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/release-candidates.tsv), and [selection records](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/release-selection.tsv). For subsequent versions, use the scan evidence linked from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest).
## Help shape what comes next
The next idea could be yours. We invite the OSS community to bring ideas, tricky integration questions, and feedback to [our Discord community](https://discord.gg/KguGXxg2X). When a direct conversation works better, reach out to **DeusData** or the project's co-developers there. Let's build what comes next together.
## License
MIT
+1 -1
View File
@@ -111,7 +111,7 @@ This project implements multiple layers of security verification. Every release
- **All dangerous function calls** require a reviewed entry in `scripts/security-allowlist.txt`
- **Time-bomb pattern detection** — scans for `time()`/`sleep()` near dangerous calls (could indicate delayed activation)
- **MCP tool handler file read audit** — tracks file read count in `mcp.c` against an expected maximum (detects added file reads that could exfiltrate data through tool responses)
- **CodeQL SAST** — static application security testing on every push (taint analysis, CWE detection, data flow tracking). Any open alert blocks the release.
- **CodeQL SAST** — static application security testing of the C/C++, JavaScript/TypeScript, Python and GitHub Actions code (taint analysis, CWE detection, data flow tracking): a full scan on every push to main, weekly, and in every dry run and release; a first-party scan of the languages a pull request changes, with alerts reported on its changed lines. Any open alert fails the run and blocks the release.
- **Fuzz testing** — random/mutated inputs to MCP server and Cypher parser (60 seconds per build). Catches crashes, segfaults, and memory errors that structured tests miss.
- **Native antivirus scanning** on every platform (any detection fails the build):
- **Windows**: Windows Defender with ML heuristics — the same engine end users run
+39 -35
View File
@@ -1,49 +1,53 @@
# CBM Lens Beta README captures
These are screenshots and browser recordings of the application, captured on 12–13 October 2026. The application data and text were not retouched. Bernhard Jackiewicz captured the recordings and PNG stills on 12 October. The Explorer and Galaxy JPEG stills were refreshed on 13 October. One addition is drawn on top of the recordings: a small mouse pointer and click ring, because a headless browser shows no pointer of its own.
These are screenshots and browser recordings of the real application using public source from this repository. The application data and text were not retouched. The GIFs were refreshed on 20 October 2026; the still images preserve the earlier captures from 12–13 October.
## Original captures: 12 October
## High-resolution recordings: 20 October
- Repository: [DeusData/codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp)
- Branch: `main`, after [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068) brought CBM Lens Beta in
- Commit: `4d9592b5`
- Build: `scripts/build.sh --with-ui`
- Browser: headless Chromium (Playwright) on macOS, English locale, viewport 1440 × 900, device scale factor 1
- Agents: standard build, experimental agent activity disabled
- Source commit: `4cf44147e639ae070ee44d9230244bfb9d9676e0`
- Build: `make -f Makefile.cbm cbm-with-ui`, with the standard frontend and experimental agent activity disabled
- Capture: built-in browser screenshots on macOS, English locale, native viewport **1600 × 1000**, **12 frames per second**
- Encoding: JPEG screenshot frames converted to GIF with one 256-color palette per recording, no dithering, infinite looping; no upscaling, sharpening or pointer overlays
The `v0.0.1` label in the application is the frontend package version. It is not the backend release version.
The `v0.0.1` label is the frontend package version, not the backend release version. Standard native releases include Lens; using a source build for these recordings is a capture-provenance detail, not an installation requirement.
### Data shown
### Data and interaction
The project `codebase-memory-mcp` is an export of the same public commit (`git archive`), indexed in `full` mode into an isolated local cache. No private repository was used. Because the export has no `.git` folder, Explore reports "Git unavailable" in its file impact line.
The project `codebase-memory-mcp` is a clean `git archive` export of the source commit, indexed in `full` mode into an isolated local cache. No private repository was used. Because the export has no `.git` folder, the interface reports "Git unavailable" where applicable.
The index reported 75,037 nodes and 361,349 edges. These are properties of this source scope, not a benchmark. The interface keeps its coverage and display-limit notices.
The index reported 81,190 nodes and 411,072 edges. These counts describe this source scope, not a benchmark or proof of complete repository coverage. The interface retains its coverage and display-limit notices. The clips join recorded interaction segments with pauses shortened between them; their displayed application states and answers are unchanged. Directional pulses represent indexed relationships, not observed execution.
## Refreshed stills: 13 October
| Recording | Duration | What it shows |
|-----------|----------|---------------|
| [lens-overview.gif](lens-overview.gif) | 12.5 seconds | Architecture overview, opening `src/foundation`, browsing its files, selecting `arena.c` and reading its source beside the map. |
| [lens-galaxy.gif](lens-galaxy.gif) | 12 seconds | Expanding outgoing relationships around `cbm_arena_alloc`, switching to the hierarchy and tracing the two-hop call path `cbm_arena_alloc → arena_grow → cbm_alloc`. |
| [lens-chat.gif](lens-chat.gif) | 11.5 seconds | `arena_grow` selected in Galaxy, with caller and callee questions answered directly from the indexed graph. |
All three GIFs are 1600 × 1000. Their frames and playback duration were decoded and checked after export. README and website use these same files.
### Local chat check
The optional Qwen2.5 Coder 0.5B model was downloaded and loaded through the application's own setup. Automatic explanations were disabled for this recording. Both displayed answers are deterministic graph answers, labelled "Listed from the indexed graph; not generated by the model." The clip shows one incoming `CALLS` relationship from `cbm_arena_alloc` and two outgoing `CALLS` relationships to `cbm_alloc` and `cbm_free`, alongside the other indexed relationship types. It does not demonstrate model-generated prose.
## Original still images: 12 October
Bernhard Jackiewicz captured the PNG stills on 12 October 2026, from commit `4d9592b5` on `main` after [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068). The build used `scripts/build.sh --with-ui`; headless Chromium on macOS used English locale, a 1440 × 900 viewport and device scale factor 1. Experimental agent activity was disabled.
The indexed project was a public `git archive` export of that commit in `full` mode, with 75,037 nodes and 361,349 edges. The PNG stills are 1440 × 900, except the chat still at 1440 × 1200 so the explanation and answer fit together. These are historical application views, also used as reduced-motion alternatives on the website; they are not extracted from the new GIFs.
| Still | What it shows |
|-------|---------------|
| [lens-architecture.png](lens-architecture.png) | The whole repository on the architecture map. |
| [lens-chat.png](lens-chat.png) | A local model explanation of `cbm_pipeline_pass_doclinks` and a caller answer listed from the graph. |
| [lens-system-structure.png](lens-system-structure.png) | Component groups and their dependencies in System structure, with `src` selected. |
| [lens-behavior.png](lens-behavior.png) | Possible direct calls of `compileCards`, with its declaration. |
## Refreshed still images: 13 October
`lens-explore.jpg` and `lens-galaxy.jpg` are unedited browser captures at 1280 × 720 from the locally served CBM Lens Beta UI. The running binary reports `dev`; its exact build commit is unavailable, so these captures do not certify an exact source revision. The local project is named `cbm-triage-main-20260919`; the screenshots show public CBM source and indexed relationships.
- **Explore:** `src/foundation/limits.c`, its compact file impact summary and direct dependencies (24 nodes, 26 edges). The UI's freshness, sampling and coverage caveats remain visible.
- **Galaxy:** `cbm_max_file_bytes`, both directions, all edge types, two layers (160 nodes, 218 edges).
- **[Explore](lens-explore.jpg):** `src/foundation/limits.c`, its compact file impact summary and direct dependencies (24 nodes, 26 edges). The UI's freshness, sampling and coverage notices remain visible.
- **[Galaxy](lens-galaxy.jpg):** `cbm_max_file_bytes`, both directions, all edge types, two layers (160 nodes, 218 edges).
These counts describe the displayed index snapshot, not a benchmark or proof of complete repository coverage. The original recordings and other stills are unchanged.
## Media
| File | What it shows |
|------|---------------|
| [lens-overview.gif](lens-overview.gif) | Architecture overview of the repository: zoom and a small orbit, selecting `src/pipeline`, opening that area, selecting `artifact.c` and opening its source. |
| [lens-architecture.png](lens-architecture.png) | Still image of the whole repository on the architecture map. |
| [lens-galaxy.gif](lens-galaxy.gif) | Galaxy: searching `cbm_pipeline_run`, following its outgoing calls one layer further, the hierarchy view and the path to `cbm_store_check_integrity`. |
| [lens-galaxy.jpg](lens-galaxy.jpg) | Two dependency layers around `cbm_max_file_bytes`, with direction and edge-type controls. |
| [lens-explore.jpg](lens-explore.jpg) | `src/foundation/limits.c` in the reader with its file impact summary, file tree and direct graph dependencies. |
| [lens-chat.gif](lens-chat.gif) | The chat with the local Qwen2.5 Coder 0.5B model: the explanation of `cbm_pipeline_pass_doclinks` and "Who calls cbm_pipeline_pass_doclinks?" answered from the graph. |
| [lens-chat.png](lens-chat.png) | Still image of the same explanation and listed answer. |
| [lens-system-structure.png](lens-system-structure.png) | Component groups and their dependencies in System structure, `src` selected. |
| [lens-behavior.png](lens-behavior.png) | Possible direct calls of `compileCards`, with its declaration. |
The recordings were captured as browser video, then encoded as GIF at 960 pixels wide and 10 frames per second with a palette per recording. While a search is typed, the Galaxy briefly shows "No matching indexed nodes." until the index search answers; those frames (under one second) were cut. The original PNG stills are 1440 × 900, the chat still 1440 × 1200 so that the explanation and the answer fit together. The refreshed JPEG stills are 1280 × 720. Directional pulses represent indexed relationships, not observed execution.
## Local chat check
The chat recording uses the actual browser model, downloaded through the application's own setup. The explanation lists the indexed facts and the declaration of the selected function; the one sentence after them was written by the model and is labelled as such. The caller answer is listed from the indexed graph without the model.
These counts describe the displayed index snapshot, not a benchmark or proof of complete repository coverage.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 700 KiB

After

Width:  |  Height:  |  Size: 2.2 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 MiB

After

Width:  |  Height:  |  Size: 3.4 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 MiB

After

Width:  |  Height:  |  Size: 1.8 MiB

+146 -82
View File
@@ -3,55 +3,103 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Codebase Memory — Connect your code, docs and infrastructure</title>
<meta name="description" content="Local code intelligence for developers and AI agents. Connect code, documentation and infrastructure in a persistent graph. Explore CBM Lens, query 17 MCP tools, configure agent integrations, and investigate change and test impact.">
<title>Codebase Memory — Local Code Intelligence &amp; MCP</title>
<meta name="description" content="A local MCP server for AI coding agents. Connect code, docs and infrastructure in a persistent knowledge graph, with 17 tools and native CLI access.">
<meta name="author" content="DeusData">
<meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1">
<meta name="theme-color" content="#0a0e0d">
<meta property="og:title" content="Codebase Memory — Understand the whole repository">
<meta property="og:description" content="Connect code, docs and infrastructure. Explore the graph in CBM Lens, give your agent structural context, and follow the impact of a change.">
<meta property="og:title" content="Codebase Memory — Local Code Intelligence &amp; MCP">
<meta property="og:description" content="A local MCP server for AI coding agents. Connect code, docs and infrastructure in a persistent knowledge graph, with 17 tools and native CLI access.">
<meta property="og:type" content="website">
<meta property="og:url" content="https://deusdata.github.io/codebase-memory-mcp/">
<meta property="og:image" content="https://deusdata.github.io/codebase-memory-mcp/images/lens-architecture.png">
<meta property="og:image:alt" content="CBM Lens architecture view of the codebase-memory-mcp repository">
<meta property="og:image:width" content="1440">
<meta property="og:image:height" content="900">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="Codebase Memory">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:title" content="Codebase Memory — Understand the whole repository">
<meta name="twitter:description" content="Local code intelligence for developers and AI agents. Code, docs, infrastructure, CBM Lens and experimental Smart CI.">
<meta name="twitter:title" content="Codebase Memory — Local Code Intelligence &amp; MCP">
<meta name="twitter:description" content="A local MCP server for AI coding agents. Connect code, docs and infrastructure in a persistent knowledge graph, with 17 tools and native CLI access.">
<meta name="twitter:image" content="https://deusdata.github.io/codebase-memory-mcp/images/lens-architecture.png">
<meta name="twitter:image:alt" content="CBM Lens architecture view of the codebase-memory-mcp repository">
<link rel="canonical" href="https://deusdata.github.io/codebase-memory-mcp/">
<link rel="alternate" type="text/plain" href="llms.txt" title="Codebase Memory facts and documentation">
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='6' fill='%230a0e0d'/%3E%3Cg stroke='%2333ff99' stroke-width='2'%3E%3Cpath d='M9 9L23 16L10 24Z' fill='none'/%3E%3C/g%3E%3Cg fill='%2333ff99'%3E%3Ccircle cx='9' cy='9' r='3.5'/%3E%3Ccircle cx='23' cy='16' r='3.5'/%3E%3Ccircle cx='10' cy='24' r='3.5'/%3E%3C/g%3E%3C/svg%3E">
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"name": "codebase-memory-mcp",
"alternateName": "Codebase Memory",
"applicationCategory": "DeveloperApplication",
"operatingSystem": "macOS, Linux, Windows",
"description": "Local code intelligence connecting source code, documentation and infrastructure in a persistent knowledge graph, with CBM Lens, MCP tools, agent integrations and experimental C/C++ test selection.",
"url": "https://deusdata.github.io/codebase-memory-mcp/",
"downloadUrl": "https://github.com/DeusData/codebase-memory-mcp/releases/latest",
"codeRepository": "https://github.com/DeusData/codebase-memory-mcp",
"programmingLanguage": "C",
"license": "https://opensource.org/licenses/MIT",
"isAccessibleForFree": true,
"offers": { "@type": "Offer", "price": "0", "priceCurrency": "USD" },
"featureList": [
"Persistent local graph of code, documentation and infrastructure",
"Structural, full-text and local semantic search",
"Hybrid LSP semantic type resolution for supported language families",
"17 MCP tools, CLI access and Cypher graph queries",
"CBM Lens source navigation, architecture maps and Galaxy graph",
"Documentation sections, PDF text pages and architecture decision records",
"Static infrastructure, deployment and configuration relationships",
"Cross-service and cross-repository linking",
"Change impact, coverage diagnostics and optional runtime trace ingestion",
"Experimental C/C++ test selection using static relationships and admitted per-test coverage",
"45 automatic/conditional client surfaces with client-specific setup",
"Optional browser AI with on-device inference"
],
"author": { "@type": "Organization", "name": "DeusData", "url": "https://github.com/DeusData" }
"@graph": [
{
"@type": "WebPage",
"@id": "https://deusdata.github.io/codebase-memory-mcp/#webpage",
"url": "https://deusdata.github.io/codebase-memory-mcp/",
"name": "Codebase Memory — Local Code Intelligence & MCP",
"description": "A local MCP server for AI coding agents. Connect code, docs and infrastructure in a persistent knowledge graph, with 17 tools and native CLI access.",
"inLanguage": "en",
"mainEntity": {
"@id": "https://deusdata.github.io/codebase-memory-mcp/#software"
}
},
{
"@type": "SoftwareApplication",
"name": "Codebase Memory",
"alternateName": [
"codebase-memory-mcp",
"CBM"
],
"applicationCategory": "DeveloperApplication",
"operatingSystem": "macOS, Linux, Windows",
"description": "A local MCP server for AI coding agents. Connect code, docs and infrastructure in a persistent knowledge graph, with 17 tools and native CLI access.",
"url": "https://deusdata.github.io/codebase-memory-mcp/",
"downloadUrl": "https://github.com/DeusData/codebase-memory-mcp/releases/latest",
"license": "https://opensource.org/licenses/MIT",
"isAccessibleForFree": true,
"offers": {
"@type": "Offer",
"price": "0",
"priceCurrency": "USD"
},
"featureList": [
"Persistent local graph of code, documentation and infrastructure",
"Structural, full-text and local semantic search",
"Hybrid LSP semantic type resolution for supported language families",
"17 MCP tools, CLI access and Cypher graph queries",
"Optional CBM Lens visual companion: source navigation, architecture maps and Galaxy graph (open beta)",
"Documentation sections, PDF text pages and architecture decision records",
"Static infrastructure, deployment and configuration relationships",
"Cross-service and cross-repository linking",
"Change impact, coverage diagnostics and optional runtime trace ingestion",
"Experimental C/C++ test selection using static relationships and admitted per-test coverage",
"45 automatic/conditional client surfaces with client-specific setup",
"Optional browser AI with on-device inference"
],
"author": {
"@type": "Organization",
"name": "DeusData",
"url": "https://github.com/DeusData"
},
"@id": "https://deusdata.github.io/codebase-memory-mcp/#software",
"image": "https://deusdata.github.io/codebase-memory-mcp/images/lens-architecture.png",
"mainEntityOfPage": {
"@id": "https://deusdata.github.io/codebase-memory-mcp/#webpage"
},
"isBasedOn": {
"@id": "https://deusdata.github.io/codebase-memory-mcp/#source"
}
},
{
"@type": "SoftwareSourceCode",
"@id": "https://deusdata.github.io/codebase-memory-mcp/#source",
"name": "codebase-memory-mcp",
"codeRepository": "https://github.com/DeusData/codebase-memory-mcp",
"programmingLanguage": "C",
"license": "https://opensource.org/licenses/MIT",
"targetProduct": {
"@id": "https://deusdata.github.io/codebase-memory-mcp/#software"
}
}
]
}
</script>
<style>
@@ -85,10 +133,16 @@
.nav { display: flex; align-items: center; justify-content: space-between; min-height: 76px; gap: 28px; }
.brand { display: flex; align-items: center; gap: 10px; font-weight: 600; color: var(--text); letter-spacing: -.025em; white-space: nowrap; }
.brand svg { width: 27px; height: 27px; color: var(--green); }
.nav-links { display: flex; gap: 28px; align-items: center; font-size: 13px; }
.nav-links { display: flex; gap: 24px; align-items: center; margin-left: auto; font-size: 13px; white-space: nowrap; }
.nav-links a { color: var(--muted); }
.nav-links a:hover { color: var(--green); }
.nav-links .nav-install { border: 1px solid var(--line); border-radius: 6px; padding: 8px 15px; color: var(--text); }
.nav-actions, .community-badges { display: flex; align-items: center; gap: 10px; }
.community-link, .nav-install { display: inline-flex; align-items: center; justify-content: center; gap: 7px; min-height: 40px; padding: 8px 13px; border: 1px solid var(--line); border-radius: 6px; font-size: 12px; font-weight: 500; white-space: nowrap; }
.community-link { color: var(--muted); background: var(--panel); }
.community-link svg { width: 16px; height: 16px; flex: none; color: var(--green); }
.nav-install { color: var(--green); background: var(--selected); }
.community-link:hover, .nav-install:hover { color: var(--text); border-color: var(--green); text-decoration: none; }
@media (max-width: 950px) { .nav-links { display: none; } }
.eyebrow { font-family: var(--mono); font-size: 11px; font-weight: 500; letter-spacing: .12em; text-transform: uppercase; color: var(--green); margin-bottom: 23px; }
.hero { padding: 80px 0 0; }
.hero-top { display: grid; grid-template-columns: 1fr 270px; gap: 65px; align-items: end; }
@@ -203,7 +257,6 @@
@media (max-width: 950px) {
.hero-top { grid-template-columns: 1fr; gap: 25px; }
.hero-aside { display: none; }
.nav-links { gap: 17px; }
.section-head { align-items: start; gap: 30px; }
.section-head p { max-width: 300px; }
.connection-grid { gap: 24px; }
@@ -215,12 +268,15 @@
}
@media (max-width: 680px) {
.container { width: calc(100% - 36px); }
.nav { min-height: 65px; gap: 12px; }
html { scroll-padding-top: 140px; }
.nav { display: grid; grid-template-columns: 1fr auto; gap: 10px; padding: 12px 0; }
.nav .brand { grid-column: 1; grid-row: 1; }
.nav-actions { display: contents; }
.nav-install { grid-column: 2; grid-row: 1; }
.community-badges { grid-column: 1 / -1; grid-row: 2; }
.community-link { flex: 1; }
.brand { font-size: 14px; }
.brand svg { width: 23px; }
.nav-links { gap: 12px; font-size: 12px; }
.nav-links .desktop-link { display: none; }
.nav-links .nav-install { padding: 7px 12px; }
.hero { padding-top: 50px; }
h1 { font-size: clamp(42px, 10.5vw, 64px); }
.hero-intro { font-size: 17px; }
@@ -267,10 +323,15 @@
Codebase Memory
</a>
<div class="nav-links">
<a class="desktop-link" href="#features">Capabilities</a>
<a class="desktop-link" href="#lens">CBM Lens</a>
<a class="desktop-link" href="#smart-ci">Smart CI</a>
<a href="https://github.com/DeusData/codebase-memory-mcp">GitHub ↗</a>
<a href="#features">Capabilities</a>
<a href="#lens">Lens beta</a>
<a href="#smart-ci">Smart CI</a>
</div>
<div class="nav-actions">
<div class="community-badges" role="group" aria-label="Community links">
<a class="community-link" href="https://discord.gg/KguGXxg2X" title="Join the community on Discord"><svg viewBox="0 0 16 16" aria-hidden="true" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="M3 2.5h10v8H7l-3.5 3v-3H3z"/><path d="M5.5 5.5h5M5.5 8h3"/></svg>Discord</a>
<a class="community-link" href="https://github.com/DeusData/codebase-memory-mcp" title="Star Codebase Memory on GitHub"><svg viewBox="0 0 16 16" aria-hidden="true" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="m8 1.5 2 4.1 4.5.7-3.3 3.2.8 4.5L8 11.9 4 14l.8-4.5L1.5 6.3l4.5-.7z"/></svg>Star on GitHub</a>
</div>
<a class="nav-install" href="#install">Get started</a>
</div>
</nav>
@@ -279,28 +340,29 @@
<section class="hero" id="what-is-it">
<div class="hero-top">
<div>
<p class="eyebrow">Local code intelligence · Open source</p>
<p class="eyebrow">Local MCP server · Open source</p>
<h1>Understand the<br><span>whole repository.</span></h1>
<p class="hero-intro">Your code, documentation and infrastructure, connected in one persistent graph. Give your agent structural context. Follow the evidence yourself in CBM Lens.</p>
<p class="hero-intro">Codebase Memory (CBM) is an open-source server for the Model Context Protocol (MCP). Give your AI coding agent a persistent knowledge graph of code, documentation and infrastructure. Search symbols, trace relationships and retrieve source evidence through 17 MCP tools.</p>
<p>Focused graph queries help reduce repeated code reading and token use. <a href="#token-savings">See the evidence →</a></p>
<div class="actions">
<a class="button primary" href="#install">Install Codebase Memory <span aria-hidden="true">↗</span></a>
<a class="button" href="#lens">Explore CBM Lens <span aria-hidden="true">↓</span></a>
<a class="button primary" href="#install">Install the MCP server <span aria-hidden="true">↗</span></a>
<a class="button" href="#features">Explore the MCP tools <span aria-hidden="true">↓</span></a>
</div>
<p class="hero-note">macOS · Linux · Windows &nbsp;/&nbsp; Native C · MIT licensed</p>
</div>
<div class="hero-aside">
<strong>For your agent</strong><p>Precise graph queries, source evidence and coverage checks through MCP.</p>
<strong>For you</strong><p>Source navigation, architecture maps and a visible path from question to code.</p>
<strong>For your agent</strong><p>17 MCP tools, source evidence and coverage checks, with client-specific skills, profiles and hooks.</p>
<strong>For your workflow</strong><p>Native CLI access, persistent indexes and optional visual exploration in Lens.</p>
</div>
</div>
<figure class="product-shot">
<div class="shot-bar"><span class="live">CBM LENS BETA</span><span class="shot-path">codebase-memory-mcp / architecture</span><span>LOCAL WORKSPACE</span></div>
<a href="images/lens-architecture.png" aria-label="View the full CBM Lens architecture screenshot"><img src="images/lens-architecture.png" alt="CBM Lens architecture map showing the source folders of codebase-memory-mcp" fetchpriority="high"></a>
<figcaption>CBM Lens, exploring this repository. Real product captures from October 2026. <a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/images/README.md">Capture details ↗</a></figcaption>
<div class="shot-bar"><span class="live">CBM LENS · OPEN BETA v0.0.1</span><span class="shot-path">codebase-memory-mcp / architecture</span><span>LOCAL WORKSPACE</span></div>
<a href="images/lens-overview.gif" aria-label="View the full CBM Lens architecture walkthrough"><picture><source media="(prefers-reduced-motion: reduce)" srcset="images/lens-architecture.png" width="1440" height="900"><img src="images/lens-overview.gif" width="1600" height="1000" alt="CBM Lens architecture walkthrough: open src/foundation, select arena.c and read its source beside the map" fetchpriority="high"></picture></a>
<figcaption>See the MCP server’s graph through CBM Lens, its optional visual companion. Open beta v0.0.1, captured in October 2026. <a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/images/README.md">Capture details ↗</a></figcaption>
</figure>
<div class="facts" aria-label="Project at a glance">
<div><strong>One connected graph</strong><span>Source, docs and infrastructure</span></div>
<div><strong>17 MCP tools</strong><span>Plus a CLI and CBM Lens</span></div>
<div><strong>17 MCP tools</strong><span>Plus native CLI access</span></div>
<div><strong>45 client surfaces</strong><span>Automatic or conditional setup</span></div>
<div><strong>Local processing</strong><span>Your index stays on your machine</span></div>
</div>
@@ -337,31 +399,10 @@
<p class="fine">Relationships come from supported source forms. Dynamic values and unresolved references can leave gaps; PDF extraction requires a text layer and does not perform OCR.</p>
</section>
<section class="section" id="lens">
<div class="section-head">
<div><p class="eyebrow">CBM Lens Beta</p><h2>A workspace for<br>understanding your code.</h2></div>
<p>Included in standard native releases. Open the local browser interface and move from the shape of a system to the source behind a symbol.</p>
</div>
<div class="lens-grid">
<figure><a href="images/lens-explore.jpg" aria-label="View the full CBM Lens source navigation screenshot"><img src="images/lens-explore.jpg" alt="Lens source reader beside the file impact summary and graph dependencies for limits.c" loading="lazy"></a><figcaption>Source navigation with indexed relationships and impact context.</figcaption></figure>
<div class="lens-copy">
<div><h3>Read code in context</h3><p>Browse files and symbols, inspect callers and dependencies, and keep the relevant source beside the graph.</p></div>
<div><h3>See the shape of the system</h3><p>Explore architecture, system structure and the Galaxy graph. Inspect coverage diagnostics and architecture decisions from the same workspace.</p></div>
</div>
</div>
<div class="gallery">
<figure><a href="images/lens-galaxy.jpg" aria-label="View the full Lens Galaxy screenshot"><img src="images/lens-galaxy.jpg" alt="CBM Lens Galaxy graph for exploring symbols and relationships" loading="lazy"></a><figcaption><strong>Follow the connections</strong>Search and expand the Galaxy graph.</figcaption></figure>
<figure><a href="images/lens-system-structure.png" aria-label="View the full Lens system structure screenshot"><img src="images/lens-system-structure.png" alt="CBM Lens system structure showing source groups and their dependencies" loading="lazy"></a><figcaption><strong>Inspect system structure</strong>See source groups and dependencies.</figcaption></figure>
<figure><a href="images/lens-chat.png" aria-label="View the full Lens local chat screenshot"><img src="images/lens-chat.png" alt="CBM Lens optional local chat explaining selected source using indexed facts" loading="lazy"></a><figcaption><strong>Ask about selected code</strong>Optional browser AI, with on-device inference.</figcaption></figure>
</div>
<p class="fine">Lens is served on loopback. Browser AI downloads its optional model after you enable it. Its explanations should be checked against the displayed source.</p>
<a class="source-link" href="https://github.com/DeusData/codebase-memory-mcp#cbm-lens-beta-code-visualization-and-source-navigation">Read the Lens guide and watch the walkthroughs →</a>
</section>
<section class="section" id="features">
<div class="section-head">
<div><p class="eyebrow">Code intelligence</p><h2>Ask better questions<br>of your repository.</h2></div>
<p>Use the same persistent graph through your coding agent, the CLI or Lens. Open source evidence whenever a result needs verification.</p>
<p>Give your coding agent structural context through 17 MCP tools. Query the same graph from the CLI and open source evidence whenever a result needs verification.</p>
</div>
<article class="feature-row" id="semantic-search"><h3>Find the right starting point</h3><div><p>Search by symbol, structure, text or meaning. Combine regex and graph filters, BM25 full-text search, and local semantic search to find relevant code even when the wording differs.</p><div class="tags"><span class="tag">search_graph</span><span class="tag">search_code</span><span class="tag">get_file_outline</span></div></div></article>
<article class="feature-row" id="hybrid-lsp"><h3>Follow actual relationships</h3><div><p>Tree-sitter parsing and Hybrid LSP resolution build call and dependency relationships for supported languages. Trace callers, callees, data flow and supported cross-service links; query more complex patterns with Cypher.</p><p>Language and framework coverage varies. <a href="https://github.com/DeusData/codebase-memory-mcp#language-support">Check the support matrix →</a></p></div></article>
@@ -399,8 +440,29 @@
</details>
</section>
<section class="section" id="lens">
<div class="section-head">
<div><p class="eyebrow">CBM Lens · Open beta v0.0.1</p><h2>A visual companion<br>to the MCP server.</h2></div>
<p>An optional way to explore the same graph your agent queries. Included in standard native releases, with source navigation, architecture maps and the Galaxy view.</p>
</div>
<div class="lens-grid">
<figure><a href="images/lens-explore.jpg" aria-label="View the full CBM Lens source navigation screenshot"><img src="images/lens-explore.jpg" width="1280" height="720" alt="Lens source reader beside the file impact summary and graph dependencies for limits.c" loading="lazy"></a><figcaption>Source navigation with indexed relationships and impact context.</figcaption></figure>
<div class="lens-copy">
<div><h3>Read code in context</h3><p>Browse files and symbols, inspect callers and dependencies, and keep the relevant source beside the graph.</p></div>
<div><h3>See the shape of the system</h3><p>Explore architecture, system structure and the Galaxy graph. Inspect coverage diagnostics and architecture decisions from the same workspace.</p></div>
</div>
</div>
<div class="gallery">
<figure><a href="images/lens-galaxy.gif" aria-label="View the full CBM Lens Galaxy walkthrough"><picture><source media="(prefers-reduced-motion: reduce)" srcset="images/lens-galaxy.jpg" width="1280" height="720"><img src="images/lens-galaxy.gif" width="1600" height="1000" alt="CBM Lens Galaxy walkthrough: expand cbm_arena_alloc and follow its call path through arena_grow to cbm_alloc" loading="lazy"></picture></a><figcaption><strong>Follow the connections</strong>Expand relationships and trace a call path.</figcaption></figure>
<figure><a href="images/lens-system-structure.png" aria-label="View the full Lens system structure screenshot"><img src="images/lens-system-structure.png" width="1440" height="900" alt="CBM Lens system structure showing source groups and their dependencies" loading="lazy"></a><figcaption><strong>Inspect system structure</strong>See source groups and dependencies.</figcaption></figure>
<figure><a href="images/lens-chat.gif" aria-label="View the full Lens local chat walkthrough"><picture><source media="(prefers-reduced-motion: reduce)" srcset="images/lens-chat.png" width="1440" height="1200"><img src="images/lens-chat.gif" width="1600" height="1000" alt="CBM Lens local chat walkthrough: callers and callees of arena_grow answered directly from the indexed graph" loading="lazy"></picture></a><figcaption><strong>Ask about selected code</strong>Caller and callee answers from the indexed graph.</figcaption></figure>
</div>
<p class="fine">Lens is served on loopback. Browser AI downloads its optional model after you enable it. Its explanations should be checked against the displayed source.</p>
<a class="source-link" href="https://github.com/DeusData/codebase-memory-mcp#cbm-lens-beta-code-visualization-and-source-navigation">Read the Lens guide and watch the walkthroughs →</a>
</section>
<section class="section" id="install">
<div class="section-head"><div><p class="eyebrow">Get started</p><h2>Install. Index. Explore.</h2></div><p>Standard native releases include CBM Lens. The native runtime needs no Docker or language runtime; the core indexer and graph queries need no API key.</p></div>
<div class="section-head"><div><p class="eyebrow">Get started</p><h2>Install. Index. Explore.</h2></div><p>Install the MCP server and configure your coding agent. The native runtime needs no Docker or language runtime; indexing and graph queries need no API key. Standard releases also include the optional Lens open beta.</p></div>
<div class="split">
<div><div class="terminal"><div class="terminal-head">TERMINAL · macOS / Linux</div><pre><code><span class="comment"># Install the native release and agent integrations</span>
<span class="command">curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash</span>
@@ -408,18 +470,20 @@
<span class="comment"># Restart your agent, then ask:</span>
"Index this project"
<span class="comment"># Open CBM Lens in your browser</span>
<span class="comment"># Optional: open CBM Lens (open beta)</span>
<span class="command">codebase-memory-mcp daemon start --open</span></code></pre></div><div class="install-links"><a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/install.sh">Inspect the installer ↗</a><a href="https://github.com/DeusData/codebase-memory-mcp#quick-start">Windows / PowerShell ↗</a><a href="https://github.com/DeusData/codebase-memory-mcp#installation">Other installation methods ↗</a></div></div>
<div class="install-copy"><h3>Your machine. Your index.</h3><p>Source analysis and graph storage run locally. Native releases contain the executable and authenticated release-owned assets, including the Lens frontend and agent integration templates.</p><p>Use native packages or the npm, PyPI and Go wrappers. Package wrappers use their respective runtimes. Nix and source builds also offer builds without the UI.</p><p>Choose the clients you want to configure, restart them after installation, then build your first index.</p><a class="button" href="https://github.com/DeusData/codebase-memory-mcp/releases/latest">Download the latest release <span aria-hidden="true">↗</span></a></div>
</div>
</section>
<section class="section" id="performance">
<div class="evidence" id="benchmark"><div><p class="eyebrow">Performance and research</p><h2>Useful context.<br>Measurable tradeoffs.</h2></div><div><p>Focused graph queries can reduce repeated file exploration. The result depends on the repository, the question, the agent and how much source verification the task needs.</p><p>Published benchmarks describe specific workloads, hardware and answer-quality tradeoffs. They are backend measurements, not a promise of a fixed saving or a benchmark of CBM Lens.</p><div class="evidence-links"><a href="https://github.com/DeusData/codebase-memory-mcp#performance">Results and limitations ↗</a><a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/BENCHMARK.md">Benchmark methodology ↗</a><a href="https://arxiv.org/abs/2603.27277">Research paper ↗</a></div></div></div>
<div class="evidence" id="benchmark"><div><p class="eyebrow">Performance and research</p><h2>More useful context.<br>Less repeated work.</h2></div><div><p id="token-savings">Reuse the graph to find relevant symbols, follow relationships and retrieve focused source snippets. Compact, paged results help keep repeated repository exploration out of your agent’s context window.</p><p>Our <a href="https://arxiv.org/abs/2603.27277v1">March 2026 preprint</a> reports <strong>10× lower token use</strong> and <strong>2.1× fewer tool calls</strong> across 31 repositories compared with file-by-file exploration.</p><p>These measurements cover the study’s backend workloads from March 2026. See the paper for the evaluation setup and full results.</p><div class="evidence-links"><a href="https://github.com/DeusData/codebase-memory-mcp#performance">Token-use results ↗</a><a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/BENCHMARK.md">Indexing benchmarks ↗</a><a href="https://arxiv.org/abs/2603.27277">Research paper ↗</a></div></div></div>
</section>
<section class="section" id="faq">
<div class="faq"><p class="eyebrow">A few practical details</p><h2>Before you start.</h2>
<div class="faq"><p class="eyebrow">Common questions</p><h2>Is Codebase Memory<br>right for your workflow?</h2>
<details><summary>When should I use Codebase Memory?</summary><p>Choose CBM when your coding agent needs a persistent local graph of code, docs and infrastructure. The MCP server brings structural search, source evidence, agent integrations and experimental test selection into one open-source package. The CLI provides direct access, and CBM Lens adds an optional visual workspace in open beta. Native releases run on macOS, Linux and Windows without Docker or a language runtime. <a href="#features">Explore the capabilities</a> and <a href="#smart-ci">Smart CI’s current scope</a>.</p></details>
<details><summary>How does Codebase Memory reduce AI token use?</summary><p>Your agent can query indexed relationships and retrieve the relevant symbols or source snippets instead of repeatedly exploring whole files. Compact results and pagination control how much context each query returns. The benefit depends on the task; <a href="#token-savings">explore the published token-use measurements</a>.</p></details>
<details><summary>Does Codebase Memory send my source code anywhere?</summary><p>The backend indexes source and stores and queries its graph locally. Your coding agent has its own data-handling settings. Lens can optionally download a browser model after you enable it, with inference on-device. Installers and update checks use the network to obtain releases and assets.</p></details>
<details><summary>How does the index stay current?</summary><p>Incremental indexing updates changed files. With auto-watch enabled, an eligible active MCP session can watch an already indexed project. Automatic first indexing is a separate opt-in setting. The local UI daemon is not a promise of a permanent background watcher.</p></details>
<details><summary>How much language coverage is available?</summary><p>Bundled tree-sitter grammars cover a broad range of languages and formats. Supported language families also have Hybrid LSP semantic resolution. Extraction depth, type resolution and framework relationships vary; the <a href="https://github.com/DeusData/codebase-memory-mcp#language-support">language support matrix</a> describes the details.</p></details>
@@ -436,7 +500,7 @@
<a class="source-link" href="https://github.com/DeusData/codebase-memory-mcp/releases">View all releases on GitHub →</a>
</section>
</main>
<footer><div class="container footer-inner"><div><a class="brand" href="#main">Codebase Memory</a><p>Local code intelligence. Open source, MIT licensed.</p></div><div class="footer-links"><a href="https://github.com/DeusData/codebase-memory-mcp">GitHub</a><a href="https://github.com/DeusData/codebase-memory-mcp#readme">Documentation</a><a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/SECURITY.md">Security</a><a href="llms.txt">llms.txt</a></div></div></footer>
<footer><div class="container footer-inner"><div><a class="brand" href="#main">Codebase Memory</a><p>Local code intelligence. Open source, MIT licensed.</p></div><div class="footer-links"><a href="https://discord.gg/KguGXxg2X">Discord</a><a href="https://github.com/DeusData/codebase-memory-mcp">GitHub</a><a href="https://github.com/DeusData/codebase-memory-mcp#readme">Documentation</a><a href="https://github.com/DeusData/codebase-memory-mcp/blob/main/SECURITY.md">Security</a><a href="llms.txt">llms.txt</a></div></div></footer>
<script>
(function () {
+8 -4
View File
@@ -1,6 +1,6 @@
# codebase-memory-mcp
> Local code intelligence for developers and AI agents. Connect code, documentation, configuration, and infrastructure in a persistent knowledge graph. Search for a feature, follow callers and dependencies, trace services and deployments, inspect change impact, and explore the evidence in CBM Lens. Access the same backend through MCP or the CLI. Indexing, graph queries, and semantic embeddings run locally without a hosted service or API key; Lens optionally downloads a browser model after opt-in for on-device code chat.
> Codebase Memory is an open-source local MCP server for AI coding agents. It connects code, documentation, configuration, and infrastructure in a persistent knowledge graph, with 17 MCP tools and native CLI access. Search for a feature, follow callers and dependencies, trace services and deployments, and inspect change impact. Indexing, graph queries, and semantic embeddings run locally without a hosted service or API key. CBM Lens is the optional visual companion, currently in open beta (v0.0.1); its optional browser model supports on-device code chat after an opt-in download.
## Capabilities
@@ -13,9 +13,10 @@
- Change impact: detect_changes maps diffs to affected symbols and investigates related tests with scope: "tests"; compare_graphs compares indexed snapshots.
- Smart CI (experimental): repository-specific changed-file rules select CI jobs; test-impact select combines static reachability, changed tests, and admitted per-test coverage to select tests. Narrowing supports C/C++ family code with supported test-harness conventions and an explicitly configured runner. Missing or doubtful evidence broadens the selection or runs all tests. Other indexed languages are not automatically supported for test narrowing. test-impact publish creates the baseline graph-and-coverage bundle. Start in shadow mode; consult the guide for version requirements.
- Architecture decisions and runtime evidence: manage_adr stores the project's decision record; ingest_traces adds separately collected observed call relationships alongside static analysis.
- CBM Lens Beta: local browser interface with source exploration, Galaxy graph navigation, architecture and behavior views, ADR editing, index-coverage diagnostics, daemon/system information, and optional local code chat. Static graph animations are not live execution traces. Activity overlays are experimental source-build features, not a standard-release promise.
- CBM Lens open beta (v0.0.1): optional visual companion to the MCP server, with a local browser interface with source exploration, Galaxy graph navigation, architecture and behavior views, ADR editing, index-coverage diagnostics, daemon/system information, and optional local code chat. Static graph animations are not live execution traces. Activity overlays are experimental source-build features, not a standard-release promise.
- Agent integrations: 45 automatic or conditional client surfaces. Installation configures supported MCP entries, durable instructions, skills, Scout/Verify/Auditor profiles, and applicable session, compaction, search, and read hooks. Coverage and activation vary by client and platform. Hooks provide context without blocking or replacing the user's tool call; some clients require trust or use parent-agent handoff instead.
- Persistent operation: a per-account daemon coordinates sessions and indexing. Existing indexes can stay synchronized during eligible active MCP sessions; initial auto-indexing is separately opt-in. Optional team graph artifacts share a baseline while each user keeps a local query database.
- Token-efficient retrieval: targeted graph queries, focused source snippets, and compact, paged results can reduce token use and repeated file reads during code exploration.
- Distribution: native C backend, MIT license, standard native releases for macOS, Linux, and Windows include CBM Lens; Linux portable archives provide static builds. Nix and development source builds can omit the UI. Package-manager options and release verification are documented in the README.
## Client coverage
@@ -24,7 +25,9 @@
## Measurement and limits
Published performance and token comparisons apply to specific workloads, hardware, and agent setups, not a universal percentage saving. The README Performance section preserves the historical comparison and research quality tradeoff. Check index generation, freshness, and exact path/scope coverage before relying on graph results, especially absence claims; read source when coverage is partial, skipped, excluded, stale, or unknown. Infrastructure relationships are static, and test-selection support is narrower than language-indexing support.
A March 2026 preprint across 31 real-world repositories reports 10× lower token use and 2.1× fewer tool calls compared with file-by-file exploration. These historical backend measurements cover the study’s workloads and agent setup. The paper documents the full evaluation; the README Performance section links to the research and earlier measurements.
Graph evidence depends on index generation, freshness, and exact path/scope coverage. Partial, skipped, excluded, stale, or unknown coverage requires source verification, especially for absence claims. Infrastructure relationships are static, and test-selection support is narrower than language-indexing support.
## Links
@@ -39,7 +42,8 @@ Published performance and token comparisons apply to specific workloads, hardwar
- Smart CI guide: https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/SMART_CI.md
- Configuration: https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/CONFIGURATION.md
- MCP tools: https://github.com/DeusData/codebase-memory-mcp#mcp-tools
- Performance and research limits: https://github.com/DeusData/codebase-memory-mcp#performance
- Token savings and evidence: https://deusdata.github.io/codebase-memory-mcp/#token-savings
- Performance and research: https://github.com/DeusData/codebase-memory-mcp#performance
- Benchmark report: https://github.com/DeusData/codebase-memory-mcp/blob/main/docs/BENCHMARK.md
- Research paper: https://arxiv.org/abs/2603.27277
- Security and release verification: https://github.com/DeusData/codebase-memory-mcp#security
+1 -2
View File
@@ -2,7 +2,6 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://deusdata.github.io/codebase-memory-mcp/</loc>
<lastmod>2026-06-12</lastmod>
<priority>1.0</priority>
<lastmod>2026-10-20</lastmod>
</url>
</urlset>
+30
View File
@@ -416,6 +416,33 @@ static TSNode resolve_lean_func_name(TSNode node, TSNode name) {
return cbm_find_child_by_kind(node, "ident");
}
// Haskell infix-form definition head (`a <+> b`, ``x `plus` y``): the defined
// name is the operator, not the left operand that is the head's first child. A
// backticked `infix_id` yields its inner variable (`plus`); a symbolic `operator`
// is returned as-is and parenthesised by cbm_func_name_node_text, matching the
// prefix form `(<+>) a b = ...` (#2440).
static TSNode resolve_haskell_infix_head_name(TSNode head) {
enum { INFIX_MIN_CHILDREN = 3, INFIX_OP_IDX = 1 };
TSNode op = ts_node_child_by_field_name(head, TS_FIELD("operator"));
if (ts_node_is_null(op) && ts_node_child_count(head) >= INFIX_MIN_CHILDREN) {
op = ts_node_child(head, INFIX_OP_IDX);
}
if (!ts_node_is_null(op)) {
const char *ok = ts_node_type(op);
if (strcmp(ok, "operator") == 0) {
return op;
}
if (strcmp(ok, "infix_id") == 0) {
TSNode v = cbm_find_child_by_kind(op, "variable");
if (!ts_node_is_null(v)) {
return v;
}
}
}
TSNode null_node = {0};
return null_node;
}
// Haskell: resolve function name from first named child (variable/name).
static TSNode resolve_haskell_func_name(TSNode node) {
if (ts_node_named_child_count(node) > 0) {
@@ -424,6 +451,9 @@ static TSNode resolve_haskell_func_name(TSNode node) {
if (strcmp(hk, "variable") == 0 || strcmp(hk, "name") == 0) {
return head;
}
if (strcmp(hk, "infix") == 0) {
return resolve_haskell_infix_head_name(head);
}
if (ts_node_named_child_count(head) > 0) {
TSNode v = ts_node_named_child(head, 0);
const char *vk = ts_node_type(v);
+8
View File
@@ -1213,6 +1213,14 @@ char *cbm_func_name_node_text(CBMArena *a, TSNode name_node, const char *source,
if (text && lang == CBM_LANG_NIX) {
cbm_nix_strip_attr_quotes(text);
}
/* Haskell infix-form operator definition: `a <+> b = …` resolves to the bare
* `operator` node `<+>`, while the prefix form `(<+>) a b = …` is named by its
* `prefix_id` text `(<+>)`. Parenthesise here, not in the resolver, so both
* forms mint the same name and the def name and the call-scope QN, which both
* route through this function, cannot disagree (#2440). */
if (text && lang == CBM_LANG_HASKELL && strcmp(ts_node_type(name_node), "operator") == 0) {
return cbm_arena_sprintf(a, "(%s)", text);
}
return text;
}
+56 -5
View File
@@ -32,6 +32,15 @@ Options:
--with-ui Build the web UI as a content-addressed sidecar (needs node).
--version V Stamp the version string (release venue passes the tag).
--arch ARCH Force target arch (arm64 | x86_64), e.g. under Rosetta.
--vendored-only Build only the third-party objects (make cbm-third-party)
into a clean BUILD_DIR, no binary.
--reuse-vendored Do NOT clean BUILD_DIR: build the binary on top of a
--vendored-only run with the same arguments, compiling
first-party code only. The pair exists for CodeQL's light
PR scan (codeql.yml): the first runs before tracing starts.
--retrace-from F With --reuse-vendored: F lists vendored paths (one per
line) whose objects must be rebuilt anyway (make -W), so a
vendored file the PR changes is compiled under tracing too.
-h, --help This text.
Make passthrough (VAR=VAL, forwarded verbatim):
@@ -79,6 +88,8 @@ source "$ROOT/scripts/path-safety.sh"
# containerized legs can build in their own directory instead of deleting and
# clobbering the host's native build/c artifacts.
WITH_UI=false
PHASE=all
RETRACE_FROM=""
VERSION=""
BUILD_DIR="build/c"
EXTRA_MAKE_ARGS=()
@@ -94,7 +105,13 @@ for arg in "$@"; do
--with-ui)
WITH_UI=true
;;
--version)
--vendored-only)
PHASE=vendored
;;
--reuse-vendored)
PHASE=reuse
;;
--version|--retrace-from)
prev_arg="$arg"
continue
;;
@@ -122,6 +139,8 @@ for arg in "$@"; do
# a usage error, not a silent make argument.
if [[ "${prev_arg:-}" == "--version" ]]; then
VERSION="$arg"
elif [[ "${prev_arg:-}" == "--retrace-from" ]]; then
RETRACE_FROM="$arg"
else
echo "build.sh: unexpected argument '$arg'. Please consult --help." >&2
exit 2
@@ -144,15 +163,47 @@ echo " ui=$WITH_UI version=${VERSION:-dev}"
# Verify compiler supports target arch
verify_compiler "$CC"
# Step 1: Clean C build artifacts only (not node_modules — npm ci handles that)
cbm_remove_build_dir "$ROOT" "$BUILD_DIR"
if [[ "$PHASE" != all ]] && $WITH_UI; then
echo "build.sh: --vendored-only / --reuse-vendored do not combine with --with-ui. Please consult --help." >&2
exit 2
fi
# --retrace-from: each listed path becomes `make -W <path>` (treated as changed
# now -- the retrace helper supplies direct compilation entrypoints, so nested
# includes do not depend on Makefile wildcard coverage; no timestamp race).
RETRACE_ARGS=()
if [[ -n "$RETRACE_FROM" ]]; then
if [[ "$PHASE" != reuse || ! -f "$RETRACE_FROM" ]]; then
echo "build.sh: --retrace-from needs --reuse-vendored and an existing list file. Please consult --help." >&2
exit 2
fi
while IFS= read -r retrace_path || [[ -n "$retrace_path" ]]; do
[[ -n "$retrace_path" ]] && RETRACE_ARGS+=(-W "$retrace_path")
done < "$RETRACE_FROM"
fi
# Step 1: Clean C build artifacts only (not node_modules — npm ci handles that).
# --reuse-vendored keeps the third-party objects of the --vendored-only run.
if [[ "$PHASE" == reuse ]]; then
if [[ ! -d "$BUILD_DIR" ]]; then
echo "build.sh: --reuse-vendored needs a --vendored-only run in $BUILD_DIR first." >&2
exit 2
fi
else
cbm_remove_build_dir "$ROOT" "$BUILD_DIR"
fi
# Step 2: Build (Makefile applies $ARCHFLAGS for the target arch on macOS)
if $WITH_UI; then
if [[ "$PHASE" == vendored ]]; then
make -j"$NPROC" -f Makefile.cbm cbm-third-party \
CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}"
echo "=== Third-party objects built in ${BUILD_DIR} (no binary) ==="
exit 0
elif $WITH_UI; then
make -j"$NPROC" -f Makefile.cbm cbm-with-ui \
CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}"
else
make -j"$NPROC" -f Makefile.cbm cbm \
make -j"$NPROC" -f Makefile.cbm cbm "${RETRACE_ARGS[@]+"${RETRACE_ARGS[@]}"}" \
CFLAGS_EXTRA="$CFLAGS_EXTRA" "${EXTRA_MAKE_ARGS[@]+"${EXTRA_MAKE_ARGS[@]}"}"
fi
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env bash
# codeql-retrace-vendored.sh — prebuilt C/C++ translation units to retrace,
# for the PR c-cpp CodeQL scan (.github/workflows/codeql.yml).
#
# A PR's c-cpp scan compiles the third-party objects BEFORE CodeQL starts
# tracing (scripts/build.sh --vendored-only), so vendored C stays out of the
# analysis, while ALL first-party C is traced -- except the vendored files the
# PR itself changes, including grammar/runtime wrappers. This lists their
# compilation entrypoints; scripts/build.sh --reuse-vendored --retrace-from
# <list> then rebuilds their objects under tracing. Use direct prerequisites
# for make -W: its shallow wildcard dependencies miss nested headers/sources,
# and compiler include paths can contain aliases such as ../unix/prim.c.
#
# The changed files come from the PR merge commit (HEAD^1 is the base it was
# merged onto, as in pr.yml's file-list fallback; needs fetch-depth 2) unless
# --changed FILE gives them. An undeterminable diff (no pull request, no
# parent, git failing, an unreadable FILE) is NOT an error and never narrows
# the scan: the list is empty and the scan still covers all first-party C.
#
# Usage: scripts/ci/codeql-retrace-vendored.sh [--changed FILE] OUT
# OUT is written with the entrypoints to retrace, one per line (may be
# empty). The scope test is scripts/ci/select-lanes.sh --codeql-scope c-cpp.
# Exit: 0 = list written · 2 = usage error.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
CHANGED_FILE=""
OUT=""
while [ $# -gt 0 ]; do
case "$1" in
-h | --help)
sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'
exit 0
;;
--changed)
CHANGED_FILE="${2:-}"
[ -n "$CHANGED_FILE" ] || {
echo "codeql-retrace-vendored.sh: --changed needs a file. Please consult --help." >&2
exit 2
}
shift
;;
-*)
echo "codeql-retrace-vendored.sh: unknown option '$1'. Please consult --help." >&2
exit 2
;;
*)
if [ -n "$OUT" ]; then
echo "codeql-retrace-vendored.sh: more than one output file. Please consult --help." >&2
exit 2
fi
OUT="$1"
;;
esac
shift
done
if [ -z "$OUT" ]; then
echo "codeql-retrace-vendored.sh: no output file. Please consult --help." >&2
exit 2
fi
changed=""
determined=false
if [ -n "$CHANGED_FILE" ]; then
if [ -r "$CHANGED_FILE" ]; then
changed=$(cat "$CHANGED_FILE")
determined=true
fi
elif [ "${GITHUB_EVENT_NAME:-}" = pull_request ]; then
if changed=$(git -C "$ROOT" diff --name-only --no-renames HEAD^1 HEAD 2>/dev/null); then
determined=true
fi
fi
: >"$OUT"
if ! $determined; then
echo "diff undetermined: no vendored file to retrace; the scan covers all first-party C"
exit 0
fi
printf '%s\n' "$changed" |
bash "$ROOT/scripts/ci/select-lanes.sh" --codeql-scope c-cpp |
while IFS= read -r path; do
# Native Windows Python writes CRLF; read removes only the LF.
path="${path%$'\r'}"
entrypoints=()
case "$path" in
internal/cbm/grammar_*.c | internal/cbm/ts_runtime.c)
entrypoints=("$path")
;;
vendored/mimalloc/*) entrypoints=(vendored/mimalloc/src/static.c) ;;
vendored/sqlite3/*) entrypoints=(vendored/sqlite3/sqlite3.c) ;;
vendored/tre/*) entrypoints=(vendored/tre/tre_all.c) ;;
internal/cbm/vendored/lz4/*)
entrypoints=(internal/cbm/vendored/lz4/lz4.c internal/cbm/vendored/lz4/lz4hc.c)
;;
internal/cbm/vendored/zstd/*) entrypoints=(internal/cbm/vendored/zstd/zstd.c) ;;
internal/cbm/vendored/ts_runtime/*) entrypoints=(internal/cbm/ts_runtime.c) ;;
internal/cbm/vendored/grammars/*)
family="${path#internal/cbm/vendored/grammars/}"
wrapper="internal/cbm/grammar_${family%%/*}.c"
if [ -f "$ROOT/$wrapper" ]; then
entrypoints=("$wrapper")
else
# Shared or unfamiliar grammar code can have many consumers.
entrypoints=("$ROOT"/internal/cbm/grammar_*.c)
fi
;;
internal/cbm/vendored/common/*)
entrypoints=("$ROOT"/internal/cbm/grammar_*.c)
;;
internal/cbm/vendored/* | vendored/*)
# Other vendored code is compiled with the first-party build.
entrypoints=("$path")
;;
esac
for entrypoint in "${entrypoints[@]+"${entrypoints[@]}"}"; do
entrypoint="${entrypoint#"$ROOT"/}"
if [ -f "$ROOT/$entrypoint" ]; then printf '%s\n' "$entrypoint"; fi
done
done | LC_ALL=C sort -u >"$OUT"
count=$(wc -l <"$OUT" | tr -d ' ')
echo "prebuilt translation units to retrace: $count"
cat "$OUT"
+3 -2
View File
@@ -66,12 +66,13 @@ if 'LANES' in os.environ:
single = {
'lint / lint': 'lint', 'lint / lint-mem': 'lint-mem',
'security / security-static': 'security-static',
'security / license-gate': 'license-gate', 'security / codeql-gate': 'codeql-gate',
'security / license-gate': 'license-gate',
'test / test-diag': 'diag', 'test / test-msan': 'msan',
'test / test-lsan-macos': 'lsan-macos', 'test / test-windows-guards': 'windows-guards',
'test / shard-completeness': 'shard-completeness', 'contracts': 'contracts',
}
matrix = ('test / test-unix (', 'test / test-windows (', 'test / test-tsan (', 'pr-smoke (')
matrix = ('test / test-unix (', 'test / test-windows (', 'test / test-tsan (', 'pr-smoke (',
'codeql / analyze (')
def lane_of(name):
if name in single:
return single[name]
+112 -8
View File
@@ -30,13 +30,19 @@
# replay's single process).
# scripts/ci/select-lanes.sh --list-lanes
# Every lane name, one per line (the universe ci-ok checks against).
# scripts/ci/select-lanes.sh --codeql-scope LANGUAGE [FILE]
# The paths of FILE (default stdin) in that CodeQL language's PR
# scope (c-cpp, javascript-typescript, python, actions), one per line.
# The codeql-* lanes are selected by these scopes; the interpreted
# ones follow .github/codeql/codeql-config.yml (paths/paths-ignore).
# Exit: 0 = decided · 2 = usage error.
set -euo pipefail
usage() { sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//'; }
usage() { sed -n '2,38p' "$0" | sed 's/^# \{0,1\}//'; }
FORMAT=json
MODE="select"
CODEQL_LANGUAGE=""
INPUT=""
EXPECT=""
while [ $# -gt 0 ]; do
@@ -56,6 +62,11 @@ while [ $# -gt 0 ]; do
[ $# -gt 1 ] && shift
;;
--list-lanes) MODE="list" ;;
--codeql-scope)
MODE="codeql-scope"
CODEQL_LANGUAGE="${2:-}"
[ $# -gt 1 ] && shift
;;
--expect-files)
# An empty value (a workflow whose count went missing) must fail
# loudly, never quietly skip the truncation check.
@@ -101,20 +112,29 @@ fi
# The heredoc below is the program, so stdin input is staged in a file first.
STAGED=""
if [ "$MODE" = select ] && [ -z "$INPUT" ]; then
case "$MODE:$CODEQL_LANGUAGE" in
codeql-scope:c-cpp | codeql-scope:javascript-typescript | codeql-scope:python | codeql-scope:actions | select:* | batch:* | list:*) ;;
*)
echo "select-lanes.sh: --codeql-scope takes c-cpp, javascript-typescript, python or actions. Please consult --help." >&2
exit 2
;;
esac
if { [ "$MODE" = select ] || [ "$MODE" = codeql-scope ]; } && [ -z "$INPUT" ]; then
STAGED=$(mktemp "${TMPDIR:-/tmp}/cbm-select-lanes.XXXXXX")
trap 'rm -f "$STAGED"' EXIT
cat >"$STAGED"
INPUT="$STAGED"
fi
python3 - "$MODE" "$FORMAT" "$INPUT" "$EXPECT" <<'PY'
CODEQL_CONFIG="$(cd "$(dirname "$0")/../.." && pwd)/.github/codeql/codeql-config.yml"
python3 - "$MODE" "$FORMAT" "$INPUT" "$EXPECT" "$CODEQL_CONFIG" "$CODEQL_LANGUAGE" <<'PY'
import json
import re
import sys
mode, fmt, source = sys.argv[1:4]
expect = int(sys.argv[4]) if sys.argv[4] else None
codeql_config, codeql_language = sys.argv[5:7]
# ── Path -> category (first matching rule wins; order is part of the rule) ──
GATE_WORKFLOWS = {"pr.yml", "_test.yml", "_lint.yml", "_security.yml",
@@ -192,6 +212,8 @@ def category(path):
return "grammar"
if path.startswith(".github/workflows/"):
return "ci-gate" if base in GATE_WORKFLOWS else "ci-other"
if path.startswith(".github/codeql/"):
return "ci-gate"
if path.startswith(".github/"):
if path.endswith(".md") or ("ISSUE_TEMPLATE" in path and path.endswith((".yml", ".yaml"))):
return "docs"
@@ -293,11 +315,16 @@ def tier_of(cats):
# ── Tier -> lanes (R1) ──
FULL = {"codeql-gate", "diag", "license-gate", "lint", "lint-mem", "lsan-macos",
# The PR CodeQL scans, one lane per language (pr.yml's `codeql` matrix). Each
# is selected by a change in its scope (codeql_scope below) and, like every
# lane, by any full run: an undeterminable diff scans every language.
CODEQL_LANES = {"c-cpp": "codeql-cpp", "javascript-typescript": "codeql-js",
"python": "codeql-py", "actions": "codeql-actions"}
FULL = {"diag", "license-gate", "lint", "lint-mem", "lsan-macos",
"memwaste", "msan", "pkg-wrappers", "security-static", "shard-completeness",
"smoke-mac", "smoke-ubuntu", "smoke-win", "tsan-arm", "tsan-mac", "tsan-x86",
"unix-arm64", "unix-macos-intel", "unix-macos14", "unix-x86", "windows",
"windows-guards"}
"windows-guards"} | set(CODEQL_LANES.values())
UNIVERSE = FULL | {"contracts"} # contracts: the T0a/T0b job; every test leg runs them too
LINT = {"lint", "lint-mem"}
CORE_LEGS = {"unix-x86", "unix-macos14", "windows", "shard-completeness"}
@@ -308,9 +335,9 @@ R1 = {
# The contract steps police test-infrastructure/ and lint config too.
"T0b-nonproduct": LINT | {"contracts"},
"T1-tests": LINT | CORE_LEGS,
"T2-extract": LINT | CORE_LEGS | {"codeql-gate", "tsan-x86", "smoke-ubuntu"},
"T3-mid": LINT | CORE_LEGS | {"codeql-gate", "tsan-x86", "smoke-ubuntu"},
"T4-platform": FULL - {"msan", "diag", "memwaste"},
"T2-extract": LINT | CORE_LEGS | {"tsan-x86", "smoke-ubuntu"},
"T3-mid": LINT | CORE_LEGS | {"tsan-x86", "smoke-ubuntu"},
"T4-platform": FULL - {"msan", "diag", "memwaste"} - set(CODEQL_LANES.values()),
"T5-core": FULL,
}
# Category add-ons: (categories, lanes). pkg adds nothing to a docs-only set
@@ -328,6 +355,73 @@ HARNESS = {"scripts/test.sh", "scripts/run-tests-parallel.sh", "scripts/run-test
"tests/test_main.c", "tests/test_framework.h"}
# ── CodeQL PR scope (the codeql-* lanes) ──
# c-cpp: any C/C++ the production build can compile -- first-party AND
# vendored (a vendored-only change is scanned: codeql.yml traces the vendored
# files a PR changes). Tests are not in the analysed build.
# Interpreted languages: their extensions inside the CodeQL config's
# paths minus paths-ignore (.github/codeql/codeql-config.yml, read below).
# A change to the CodeQL machinery itself scans every language (fail closed);
# those paths are full runs anyway, this keeps that true if a tier moves.
CODEQL_C_EXT = (".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".inc")
CODEQL_C_ROOTS = ("src/", "internal/", "vendored/")
CODEQL_EXT = {"javascript-typescript": (".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx",
".mts", ".cts", ".vue", ".html", ".htm"),
"python": (".py",)}
CODEQL_FAIL_CLOSED_PREFIXES = (".github/workflows/", ".github/codeql/")
CODEQL_FAIL_CLOSED = {"Makefile.cbm", "scripts/build.sh", "scripts/ci/select-lanes.sh",
"scripts/ci/codeql-retrace-vendored.sh"}
def config_lists(path):
"""paths / paths-ignore of the CodeQL config: `key:` then ` - item` lines.
None when the file is unreadable -- every interpreted scope then matches
by extension alone (fail closed)."""
try:
lines = open(path, encoding="utf-8").read().splitlines()
except OSError:
return None
lists, key = {"paths": [], "paths-ignore": []}, None
for line in lines:
if not line.strip() or line.lstrip().startswith("#"):
continue
m = re.match(r"^([A-Za-z-]+):\s*$", line)
if m:
key = m.group(1) if m.group(1) in lists else None
continue
m = re.match(r"^\s+-\s+['\"]?([^'\"#]+?)['\"]?\s*$", line)
if m and key:
lists[key].append(m.group(1).rstrip("/"))
elif not line.startswith((" ", "\t")):
key = None
return lists
CODEQL_PATHS = config_lists(codeql_config)
def under(path, prefixes):
return any(path == p or path.startswith(p + "/") for p in prefixes)
def codeql_fail_closed(path):
return path in CODEQL_FAIL_CLOSED or path.startswith(CODEQL_FAIL_CLOSED_PREFIXES)
def codeql_scope(language, path):
if codeql_fail_closed(path):
return True
if language == "c-cpp":
return path.startswith(CODEQL_C_ROOTS) and path.endswith(CODEQL_C_EXT)
if CODEQL_PATHS is not None and (not under(path, CODEQL_PATHS["paths"])
or under(path, CODEQL_PATHS["paths-ignore"])):
return False
if language == "actions":
return path.startswith(".github/workflows/") or \
path.rsplit("/", 1)[-1] in ("action.yml", "action.yaml")
return path.endswith(CODEQL_EXT[language])
def triggers(path, cat):
found = []
if path == "scripts/ci/select-lanes.sh":
@@ -380,6 +474,12 @@ def select(paths, expect=None):
lanes = lanes | extra
if lanes & TEST_LEGS:
lanes = lanes | {"shard-completeness"}
for language, lane in sorted(CODEQL_LANES.items()):
hits = [p for p in paths if codeql_scope(language, p)]
if hits and lane not in lanes:
more = f" (+{len(hits) - 1} more)" if len(hits) > 1 else ""
reasons.append(f"codeql {language}: {hits[0]}{more}")
lanes = lanes | {lane}
for trigger, hits in by_trigger.items():
more = f" (+{len(hits) - 1} more)" if len(hits) > 1 else ""
reasons.append(f"full: {trigger}: {hits[0]}{more}")
@@ -407,6 +507,10 @@ def read_paths(path):
if mode == "list":
print("\n".join(sorted(UNIVERSE)))
elif mode == "codeql-scope":
scoped = sorted({p for p in read_paths(source) if p and codeql_scope(codeql_language, p)})
if scoped:
print("\n".join(scoped))
elif mode == "batch":
with open(source, encoding="utf-8") as fh:
for line in fh:
@@ -2,12 +2,14 @@
# pushes, from the lane analysis ground truth: which lane caught each. The
# replay asserts the selector selects that lane for that push.
# Columns (TAB): run_id, attempt, pr, head_sha, lane, classification.
# The three codeql-action-bump failures were caught by the codeql-gate lane;
# CodeQL now runs inside PR CI as the codeql-* lanes, so their lane is codeql-cpp.
34263688699 1 2115 013e1f7ec7d4b981a1cd28bf35d5767c195f8447 unix-arm64 real: cli suite hang on slow runners (arm64, then intel)
34288397854 1 2115 92cb3b9a4bebbc79bd283e447104669ca068ea64 unix-arm64 real: cli suite hang on slow runners (arm64, then intel)
34649406520 1 2115 78a6d19f1eaaa73054639ab0f14c372c53cd1bb3 unix-arm64 real: cli suite hang on slow runners (arm64, then intel)
34649406520 2 2115 78a6d19f1eaaa73054639ab0f14c372c53cd1bb3 unix-arm64 real: cli suite hang on slow runners (arm64, then intel)
34676336284 1 2185 eb5e34cfc141454e849daf413f648e73f74c455a codeql-gate real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze)
34676429093 1 2187 cea35fcc0520a0ba95139bd0210052f09b9cfa20 codeql-gate real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze)
34676336284 1 2185 eb5e34cfc141454e849daf413f648e73f74c455a codeql-cpp real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze)
34676429093 1 2187 cea35fcc0520a0ba95139bd0210052f09b9cfa20 codeql-cpp real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze)
34696991657 1 2115 9a460b389ce3002ac164605c3627bb86aa3b9be3 unix-macos-intel real: cli suite hang on slow runners (arm64, then intel)
34805101784 1 2202 7b77fd4834afe355cc4c7326ac3a55f0502f7264 lsan-macos real: macOS-only LSan leaks (memory-core)
34805101784 1 2202 7b77fd4834afe355cc4c7326ac3a55f0502f7264 msan real?: MSan stack overflow (memory-core, branch-specific)
@@ -23,4 +25,4 @@
35483768695 1 2236 d2dd3b347bbdcc695f911e80b1b7f7a32d790e5a smoke-mac real: cache CLI broken on macOS/Windows
35483768695 1 2236 d2dd3b347bbdcc695f911e80b1b7f7a32d790e5a smoke-win real: cache CLI broken on macOS/Windows
35493947102 1 2236 2caafc0c41bd083092afeb692824e718a8e1e007 smoke-mac real: cache CLI broken on macOS/Windows
36242675954 1 2363 363575af920b05f911fc76ca964dcfa7ef1c62bb codeql-gate real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze)
36242675954 1 2363 363575af920b05f911fc76ca964dcfa7ef1c62bb codeql-cpp real: codeql-action bump broke CodeQL (still caught by requiring CodeQL/analyze)
1 # The 22 real (non-flake, non-infra) job failures of the September 2026 PR
2 # pushes, from the lane analysis ground truth: which lane caught each. The
3 # replay asserts the selector selects that lane for that push.
4 # Columns (TAB): run_id, attempt, pr, head_sha, lane, classification.
5 # The three codeql-action-bump failures were caught by the codeql-gate lane;
6 # CodeQL now runs inside PR CI as the codeql-* lanes, so their lane is codeql-cpp.
7 34263688699
8 34288397854
9 34649406520
10 34649406520
11 34676336284
12 34676429093
13 34696991657
14 34805101784
15 34805101784
25 35483768695
26 35483768695
27 35493947102
28 36242675954
+2
View File
@@ -383,6 +383,8 @@ bash "$ROOT/tests/test_setup_scripts_contract.sh"
# every September 2026 PR push and real failure against it.
echo "=== Step 0z: PR lane selector contract ==="
bash "$ROOT/tests/test_select_lanes.sh"
echo "=== Step 0z1: CodeQL retraces affected prebuilt objects ==="
bash "$ROOT/tests/test_codeql_retrace.sh"
echo "=== Step 0z2: lane selector history replay ==="
bash "$ROOT/scripts/test-impact/replay-selector.sh"
echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ==="
+14 -3
View File
@@ -2899,12 +2899,23 @@ static cyp_partial_t partial_or(cyp_partial_t l, cyp_partial_t r) {
: CYP_PARTIAL_UNKNOWN;
}
/* A condition whose alias is not bound yet is UNKNOWN; a literal-only LHS
* (func condition with no variable arg) has nothing to wait for. */
static bool partial_alias_unbound(binding_t *b, const char *alias) {
return alias && !binding_get(b, alias) && !binding_get_edge(b, alias);
}
/* A condition is UNKNOWN while any alias it reads is not bound yet: its own
* variable, and every function argument's -- a func condition is filed under
* its first argument's alias, so the b in coalesce(a.x, b.name) is only seen
* here. A literal-only LHS has nothing to wait for. */
static cyp_partial_t eval_condition_partial(const cbm_condition_t *c, binding_t *b) {
if (c->variable && !binding_get(b, c->variable) && !binding_get_edge(b, c->variable)) {
if (partial_alias_unbound(b, c->variable)) {
return CYP_PARTIAL_UNKNOWN;
}
for (int i = 0; i < c->arg_count; i++) {
if (partial_alias_unbound(b, c->args[i].variable)) {
return CYP_PARTIAL_UNKNOWN;
}
}
return eval_condition(c, b) ? CYP_PARTIAL_TRUE : CYP_PARTIAL_FALSE;
}
+2 -2
View File
@@ -449,8 +449,8 @@ static bool application_regular_db_exists(const char *project) {
if (written <= 0 || (size_t)written >= sizeof(path)) {
return false;
}
struct stat status;
return stat(path, &status) == 0 && S_ISREG(status.st_mode);
cbm_path_info_t status;
return cbm_path_info_utf8(path, &status) == CBM_PATH_INFO_OK && status.is_regular;
}
static bool application_canonical_directory_exists(const char *path) {
+27 -9
View File
@@ -522,14 +522,28 @@ static watcher_git_status_t git_repo_status(cbm_watcher_t *w, project_state_t *s
* a `.git` file (the gitlink form used by linked worktrees and initialized
* submodules). Deliberately a filesystem check, not a git invocation — the whole
* point is to learn something `rev-parse` cannot tell us, because it walks up. */
/* Preserve stat's link-following behavior while inspecting UTF-8 paths. */
static int watcher_path_info(const char *path, cbm_path_info_t *info) {
int status = cbm_path_info_utf8(path, info);
if (status == CBM_PATH_INFO_OK && info->is_symlink) {
char canonical[CBM_SZ_4K];
if (!cbm_canonical_path(path, canonical, sizeof(canonical))) {
return CBM_PATH_INFO_UNAVAILABLE;
}
status = cbm_path_info_utf8(canonical, info);
}
return status;
}
static bool git_has_own_dot_git(const char *root_path) {
char path[CBM_SZ_4K];
int written = snprintf(path, sizeof(path), "%s/.git", root_path);
if (written <= 0 || (size_t)written >= sizeof(path)) {
return false;
}
struct stat st;
return stat(path, &st) == 0 && (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode));
cbm_path_info_t info;
return watcher_path_info(path, &info) == CBM_PATH_INFO_OK &&
(info.is_directory || info.is_regular);
}
/* Does the ancestor repository actually track anything inside this directory?
@@ -990,13 +1004,17 @@ static root_status_t root_status(const char *root_path, int *out_errno) {
if (!root_path) {
return ROOT_UNCERTAIN;
}
struct stat st;
if (stat(root_path, &st) == 0) {
/* Exists but is no longer a directory → the root directory is gone. */
return S_ISDIR(st.st_mode) ? ROOT_PRESENT : ROOT_MISSING;
cbm_path_info_t info;
int status = watcher_path_info(root_path, &info);
if (status == CBM_PATH_INFO_OK) {
return info.is_directory ? ROOT_PRESENT : ROOT_MISSING;
}
#ifdef _WIN32
*out_errno = status == CBM_PATH_INFO_ABSENT ? ENOENT : EIO;
#else
*out_errno = errno;
return cbm_watcher_root_missing_errno(errno) ? ROOT_MISSING : ROOT_UNCERTAIN;
#endif
return status == CBM_PATH_INFO_ABSENT ? ROOT_MISSING : ROOT_UNCERTAIN;
}
/* Sustained-absence window (seconds) before a missing root may be pruned.
@@ -1338,8 +1356,8 @@ static void publish_status(project_state_t *s, bool scanned) {
/* Init baseline for a project: check if git, get HEAD, count files */
static bool init_baseline(cbm_watcher_t *w, project_state_t *s) {
struct stat st;
if (stat(s->root_path, &st) != 0) {
cbm_path_info_t info;
if (watcher_path_info(s->root_path, &info) != CBM_PATH_INFO_OK || !info.is_directory) {
cbm_log_warn("watcher.root_gone", "project", s->project_name, "path", s->root_path);
s->baseline_done = true;
s->is_git = false;
+20 -16
View File
@@ -10,7 +10,7 @@ npm-readme-runs-the-test-legs pkg/npm/README.md T1-tests false lint lint-mem sec
issue-template-is-docs .github/ISSUE_TEMPLATE/bug.yml T0a-docs false contracts security-static
local-infra test-infrastructure/run.sh T0b-nonproduct false contracts lint lint-mem security-static
lint-config scripts/memory-core-baseline.txt T0b-nonproduct false contracts lint lint-mem security-static
graph-ui-adds-guards graph-ui/src/App.tsx T0b-nonproduct false contracts lint lint-mem security-static smoke-ubuntu windows-guards
graph-ui-adds-guards graph-ui/src/App.tsx T0b-nonproduct false contracts lint lint-mem security-static smoke-ubuntu windows-guards codeql-js
memwaste-tools-add-memwaste scripts/memwaste.sh T0b-nonproduct false contracts lint lint-mem memwaste security-static
non-gating-script-runs-contracts-leg scripts/benchmark-index.sh T0b-nonproduct false contracts lint lint-mem security-static shard-completeness unix-x86
test-impact-tooling scripts/test-impact/replay-selector.sh T0b-nonproduct false contracts lint lint-mem security-static shard-completeness unix-x86
@@ -19,19 +19,19 @@ test-fixture tests/fixtures/cpp_include/a.h T1-tests false lint lint-mem securit
contract-test tests/test_version_metadata_contract.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows
fuzz-harness tests/fuzz/fuzz_cypher.c T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows
docs-plus-tests README.md tests/test_store.c T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows
lsp-language internal/cbm/lsp/go_lsp.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
lsp-core internal/cbm/lsp/type_registry.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
grammar-is-extract-not-full internal/cbm/vendored/grammars/go/parser.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
extraction internal/cbm/extract_calls.c T2-extract false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
pipeline src/pipeline/pass_calls.c T3-mid false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
mcp-plus-tests src/mcp/mcp.c tests/test_mcp.c T3-mid false codeql-gate lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
mid-plus-memwaste-tools src/store/store.c scripts/memwaste-report.py T3-mid false codeql-gate lint lint-mem memwaste security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows
daemon src/daemon/application.c T4-platform false FULL -msan -diag -memwaste
main-c-is-cli src/main.c T4-platform false FULL -msan -diag -memwaste
foundation-non-memory src/foundation/log.c T4-platform false FULL -msan -diag -memwaste
windows-guard-tests tests/windows/guard.ps1 T4-platform false FULL -msan -diag -memwaste
smoke-harness scripts/smoke-local.sh T4-platform false FULL -msan -diag -memwaste
package-wrappers pkg/npm/package.json T4-platform false FULL -msan -diag -memwaste
lsp-language internal/cbm/lsp/go_lsp.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp
lsp-core internal/cbm/lsp/type_registry.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp
grammar-is-extract-not-full internal/cbm/vendored/grammars/go/parser.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp
extraction internal/cbm/extract_calls.c T2-extract false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp
pipeline src/pipeline/pass_calls.c T3-mid false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp
mcp-plus-tests src/mcp/mcp.c tests/test_mcp.c T3-mid false lint lint-mem security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp
mid-plus-memwaste-tools src/store/store.c scripts/memwaste-report.py T3-mid false lint lint-mem memwaste security-static shard-completeness smoke-ubuntu tsan-x86 unix-macos14 unix-x86 windows codeql-cpp codeql-py
daemon src/daemon/application.c T4-platform false FULL -msan -diag -memwaste -codeql-js -codeql-py -codeql-actions
main-c-is-cli src/main.c T4-platform false FULL -msan -diag -memwaste -codeql-js -codeql-py -codeql-actions
foundation-non-memory src/foundation/log.c T4-platform false FULL -msan -diag -memwaste -codeql-js -codeql-py -codeql-actions
windows-guard-tests tests/windows/guard.ps1 T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions
smoke-harness scripts/smoke-local.sh T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions
package-wrappers pkg/npm/package.json T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions
memory-core src/foundation/mem_core.c T5-core true FULL
gate-script scripts/ci/require-all-green.sh T5-core true FULL
the-selector-itself scripts/ci/select-lanes.sh T5-core true FULL
@@ -50,8 +50,8 @@ lint-config-file .cppcheck T0b-nonproduct false contracts lint lint-mem security
script-c-tests-read scripts/security-vendored.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows
engine-config-c-tests-read .codebase-memory.json T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows
test-failure-report scripts/suite-failure-report.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows
smoke-security-script scripts/security-strings.sh T4-platform false FULL -msan -diag -memwaste
shipped-notices-are-pkg scripts/gen-third-party-notices.sh T4-platform false FULL -msan -diag -memwaste
smoke-security-script scripts/security-strings.sh T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions
shipped-notices-are-pkg scripts/gen-third-party-notices.sh T4-platform false FULL -msan -diag -memwaste -codeql-cpp -codeql-js -codeql-py -codeql-actions
non-gate-workflow-forces-full .github/workflows/stale.yml T0b-nonproduct true FULL
harness-main-forces-full tests/test_main.c T1-tests true FULL
harness-framework-forces-full tests/test_framework.h T1-tests true FULL
@@ -59,3 +59,7 @@ harness-driver-forces-full scripts/test.sh T5-core true FULL
harness-scheduler-forces-full scripts/run-test-wave.py T5-core true FULL
lower-tier-plus-trigger-is-full docs/x.md src/pipeline/a.c tests/test_main.c T3-mid true FULL
empty-list-fails-safe - T0a-docs true FULL
graph-ui-python-agent graph-ui/agents/agent.py T0b-nonproduct false codeql-py contracts lint lint-mem security-static smoke-ubuntu windows-guards
vendored-js-is-not-scanned graph-ui/vendor/lib.js T0b-nonproduct false contracts lint lint-mem security-static smoke-ubuntu windows-guards
vendored-only-c-scans-c vendored/sqlite3/sqlite3.c T5-core true FULL
codeql-config-is-full .github/codeql/codeql-config.yml T5-core true FULL
Can't render this file because it contains an unexpected character in line 3 and column 58.
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# Exercise the real production Make rules with tiny C fixtures. CodeQL starts
# tracing after cbm-third-party, so every affected object must compile again,
# while unrelated objects must remain reused. No dry-run/timestamp assertions.
set -euo pipefail
ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}"
python3 - "$ROOT" "$BASH" <<'PY'
import os
import pathlib
import re
import subprocess
import sys
import tempfile
root, bash = pathlib.Path(sys.argv[1]), sys.argv[2]
failures = []
with tempfile.TemporaryDirectory(prefix="cbm-codeql-retrace-") as directory:
work = pathlib.Path(directory)
def write(path, text):
dest = work / path
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(text, encoding="utf-8")
for path in ("Makefile.cbm", "scripts/ci/codeql-retrace-vendored.sh",
"scripts/ci/select-lanes.sh", ".github/codeql/codeql-config.yml"):
write(path, (root / path).read_text(encoding="utf-8"))
# Nested and aliased includes deliberately escape the Makefile's shallow
# wildcard dependencies. The compiler really compiles these translation units.
write("vendored/mimalloc/src/static.c",
'#include "mimalloc/internal.h"\n#include "prim/osx/prim.c"\n')
write("vendored/mimalloc/include/mimalloc/internal.h", "#define MI_VALUE 7\n")
write("vendored/mimalloc/src/prim/osx/prim.c", '#include "../unix/prim.c"\n')
write("vendored/mimalloc/src/prim/unix/prim.c", "int mi_value(void) { return MI_VALUE; }\n")
write("internal/cbm/vendored/common/scanner.h", "#define SHARED_VALUE 3\n")
for language in ("go", "json"):
write(f"internal/cbm/grammar_{language}.c",
f'#include "vendored/grammars/{language}/parser.c"\n')
write(f"internal/cbm/vendored/grammars/{language}/parser.c",
'#include "nested/value.h"\n#include "../../common/scanner.h"\n'
f'int grammar_{language}(void) {{ return VALUE + SHARED_VALUE; }}\n')
write(f"internal/cbm/vendored/grammars/{language}/nested/value.h", "#define VALUE 2\n")
write("internal/cbm/vendored/grammars/shared/shared.h", "/* future shared grammar helper */\n")
write("internal/cbm/ts_runtime.c", '#include "vendored/ts_runtime/src/lib.c"\n')
write("internal/cbm/vendored/ts_runtime/src/lib.c", "int runtime_value(void) { return 1; }\n")
for path, symbol in (("vendored/sqlite3/sqlite3.c", "sqlite_value"),
("vendored/tre/tre_all.c", "tre_value"),
("internal/cbm/vendored/lz4/lz4.c", "lz4_value"),
("internal/cbm/vendored/lz4/lz4hc.c", "lz4hc_value"),
("internal/cbm/vendored/zstd/zstd.c", "zstd_value")):
write(path, f"int {symbol}(void) {{ return 1; }}\n")
write("internal/cbm/vendored/lz4/lz4.h", "/* shared LZ4 header */\n")
write("vendored/sqlite3/sqlite3.h", "/* sqlite header */\n")
write("internal/cbm/vendored/zstd/zstd.h", "/* zstd header */\n")
write("vendored/nomic/code_vectors_blob.S", ".text\n")
write("vendored/nomic/code_vectors.bin", "")
write("src/main.c", "int main(void) { return 0; }\n")
def run(args):
proc = subprocess.run(args, cwd=work, capture_output=True, text=True)
if proc.returncode:
raise RuntimeError(f"{args}: exit {proc.returncode}\n{proc.stdout}\n{proc.stderr}")
return proc.stdout
make = ["make", "--no-print-directory", "-f", "Makefile.cbm", "cbm-third-party",
"BUILD_DIR=build/c", "CC=" + os.environ.get("CC", "cc")]
def compiled(output):
return set(re.findall(r" -c -o build/c/([^\s]+\.o)(?:\s|$)", output))
prebuilt = compiled(run(make))
expected = {"prod_mimalloc.o", "prod_sqlite3.o", "prod_grammar_go.o",
"prod_grammar_json.o", "prod_ts_runtime.o", "prod_lz4.o",
"prod_lz4hc.o", "prod_zstd.o", "unixcoder_blob.o"}
# TRE is a Windows-only production object. All other objects are mandatory.
if prebuilt not in (expected, expected | {"prod_tre.o"}):
raise RuntimeError(f"incomplete prebuild: {sorted(prebuilt)}")
if compiled(run(make)):
raise RuntimeError("unchanged prebuild is not reused")
cases = [
("nested allocator header", ["vendored/mimalloc/include/mimalloc/internal.h"], {"prod_mimalloc.o"}),
("aliased allocator source", ["vendored/mimalloc/src/prim/unix/prim.c"], {"prod_mimalloc.o"}),
("nested grammar header", ["internal/cbm/vendored/grammars/go/nested/value.h"], {"prod_grammar_go.o"}),
("grammar wrapper", ["internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),
("runtime wrapper", ["internal/cbm/ts_runtime.c"], {"prod_ts_runtime.o"}),
("runtime source", ["internal/cbm/vendored/ts_runtime/src/lib.c"], {"prod_ts_runtime.o"}),
("shared grammar header", ["internal/cbm/vendored/common/scanner.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),
("unknown grammar family", ["internal/cbm/vendored/grammars/shared/shared.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),
("LZ4 header", ["internal/cbm/vendored/lz4/lz4.h"], {"prod_lz4.o", "prod_lz4hc.o"}),
("SQLite header", ["vendored/sqlite3/sqlite3.h"], {"prod_sqlite3.o"}),
("zstd header", ["internal/cbm/vendored/zstd/zstd.h"], {"prod_zstd.o"}),
("first-party only", ["src/main.c"], set()),
("duplicate changes", ["internal/cbm/grammar_go.c", "internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),
]
for label, paths, expected_objects in cases:
write("changed.txt", "\n".join(paths) + "\n")
run([bash, "scripts/ci/codeql-retrace-vendored.sh", "--changed", "changed.txt", "retrace.txt"])
retrace = (work / "retrace.txt").read_text(encoding="utf-8").splitlines()
actual = compiled(run(make + [arg for path in retrace for arg in ("-W", path)]))
if actual != expected_objects:
failures.append(f"{label}: expected {sorted(expected_objects)}, compiled {sorted(actual)}")
else:
print(f"PASS {label}: {len(actual)}/{len(prebuilt)} third-party objects retraced")
if failures:
print("CODEQL RETRACE CONTRACT VIOLATED:\n " + "\n ".join(failures))
sys.exit(1)
print("CodeQL retrace contract OK (real compiler + production Make rules)")
PY
+111
View File
@@ -1917,6 +1917,32 @@ TEST(cypher_exists_has_outgoing_calls) {
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 2);
cbm_cypher_result_free(&r);
rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WHERE b.name <> \"SubmitOrder\" "
"RETURN a.name, b.name",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 2);
cbm_cypher_result_free(&r);
rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WHERE b.name = \"SubmitOrder\" OR "
"NOT b.name = \"SubmitOrder\" RETURN count(*)",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 1);
ASSERT_STR_EQ(r.rows[0][0], "3");
cbm_cypher_result_free(&r);
rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WITH a, b WHERE NOT b.name = "
"\"SubmitOrder\" RETURN a.name, b.name",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 2);
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
@@ -3378,6 +3404,86 @@ TEST(cypher_exec_where_mixed_alias_xor) {
PASS();
}
TEST(cypher_exec_where_not_target_node) {
cbm_store_t *s = setup_cypher_store();
cbm_cypher_result_t r = {0};
int rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WHERE NOT b.name = \"SubmitOrder\" "
"RETURN a.name, b.name",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 2);
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
TEST(cypher_exec_where_not_target_relationship) {
cbm_store_t *s = setup_cypher_multi_edge_store();
cbm_cypher_result_t r = {0};
int rc = cbm_cypher_execute(s,
"MATCH (a)-[r:HTTP_CALLS]->(b) WHERE NOT r.url_path = "
"\"/health\" RETURN r.url_path",
"testproj", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 1);
ASSERT_STR_EQ(r.rows[0][0], "/api/orders");
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
TEST(cypher_exec_where_not_source_node) {
cbm_store_t *s = setup_cypher_store();
cbm_cypher_result_t r = {0};
int rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WHERE NOT a.name = "
"\"HandleOrder\" RETURN a.name, b.name",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 1);
ASSERT_STR_EQ(r.rows[0][0], "ValidateOrder");
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
TEST(cypher_exec_where_not_target_node_count) {
cbm_store_t *s = setup_cypher_store();
cbm_cypher_result_t r = {0};
int rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WHERE NOT b.name = "
"\"SubmitOrder\" RETURN count(*)",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 1);
ASSERT_STR_EQ(r.rows[0][0], "2");
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
/* The early WHERE pass must wait for every alias a condition reads. A function
* condition is filed under its first argument's alias, so in
* coalesce(a.no_such_prop, b.name) the target b hid behind the bound a: the
* seed row read coalesce(NULL, <unbound b>) as a definite mismatch and pruned
* every row before CALLS expansion, while the same predicate after WITH was
* right. */
TEST(cypher_exec_where_func_arg_on_unbound_target) {
cbm_store_t *s = setup_cypher_store();
cbm_cypher_result_t r = {0};
int rc = cbm_cypher_execute(s,
"MATCH (a)-[:CALLS]->(b) WHERE coalesce(a.no_such_prop, b.name) = "
"\"SubmitOrder\" RETURN a.name",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 1);
ASSERT_STR_EQ(r.rows[0][0], "ValidateOrder");
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
TEST(cypher_exec_where_in) {
cbm_store_t *s = setup_cypher_store();
cbm_cypher_result_t r = {0};
@@ -6243,6 +6349,11 @@ SUITE(cypher) {
RUN_TEST(cypher_exec_where_mixed_alias_and);
RUN_TEST(cypher_exec_where_mixed_alias_or);
RUN_TEST(cypher_exec_where_mixed_alias_xor);
RUN_TEST(cypher_exec_where_not_target_node);
RUN_TEST(cypher_exec_where_not_target_relationship);
RUN_TEST(cypher_exec_where_not_source_node);
RUN_TEST(cypher_exec_where_not_target_node_count);
RUN_TEST(cypher_exec_where_func_arg_on_unbound_target);
RUN_TEST(cypher_exec_where_in);
RUN_TEST(cypher_exec_where_not_in);
RUN_TEST(cypher_exec_where_is_null);
+28
View File
@@ -1777,6 +1777,33 @@ TEST(haskell_function) {
PASS();
}
/* #2440: an infix-form definition (`a <+> b = ...`, ``x `plus` y = ...``) has
* no `name` field; its first named child is an `infix` node whose first child
* is the LEFT OPERAND, so the def was named `a` / `x`. It must be named after
* the operator, consistent with the prefix form `(<+>) a b = ...`. */
TEST(haskell_infix_definition_name) {
CBMFileResult *r = extract("module M where\n"
"\n"
"(<+>) :: Int -> Int -> Int\n"
"a <+> b = a + b\n"
"\n"
"plus :: Int -> Int -> Int\n"
"x `plus` y = x + y\n"
"\n"
"f :: Int\n"
"f = 1 <+> 2\n",
CBM_LANG_HASKELL, "t", "M.hs");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Function", "f"));
ASSERT(has_def(r, "Function", "plus"));
ASSERT(has_def(r, "Function", "(<+>)"));
ASSERT_FALSE(has_def(r, "Function", "a"));
ASSERT_FALSE(has_def(r, "Function", "x"));
cbm_free_result(r);
PASS();
}
/* --- OCaml --- */
TEST(ocaml_function) {
CBMFileResult *r =
@@ -13177,6 +13204,7 @@ SUITE(extraction) {
RUN_TEST(elixir_function);
RUN_TEST(elixir_call_string_argument);
RUN_TEST(haskell_function);
RUN_TEST(haskell_infix_definition_name);
RUN_TEST(ocaml_function);
RUN_TEST(erlang_function);
-1
View File
@@ -45,7 +45,6 @@ t1_jobs() { # the jobs of a green T1 run; $1 = conclusion of the x86 leg
job "changes" success
job "security / security-static" success
job "security / license-gate" skipped
job "security / codeql-gate" skipped
job "lint / lint" success
job "lint / lint-mem" success
job "test / setup-matrix" success
+2 -2
View File
@@ -156,10 +156,10 @@ require("test_selection: ${{ needs.select-tests.result == 'success' && "
MATRIX = {"unix-x86": "_test.yml", "unix-arm64": "_test.yml", "unix-macos14": "_test.yml",
"unix-macos-intel": "_test.yml", "windows": "_test.yml", "tsan-x86": "_test.yml",
"tsan-arm": "_test.yml", "tsan-mac": "_test.yml", "smoke-ubuntu": "pr.yml",
"smoke-mac": "pr.yml", "smoke-win": "pr.yml"}
"smoke-mac": "pr.yml", "smoke-win": "pr.yml", "codeql-cpp": "pr.yml",
"codeql-js": "pr.yml", "codeql-py": "pr.yml", "codeql-actions": "pr.yml"}
SINGLE = {"lint": ("_lint.yml", "lint"), "lint-mem": ("_lint.yml", "lint-mem"),
"license-gate": ("_security.yml", "license-gate"),
"codeql-gate": ("_security.yml", "codeql-gate"),
"pkg-wrappers": ("_test.yml", "test-package-wrappers"),
"diag": ("_test.yml", "test-diag"), "msan": ("_test.yml", "test-msan"),
"lsan-macos": ("_test.yml", "test-lsan-macos"),
+3 -5
View File
@@ -46,14 +46,12 @@ if [[ "$manifest_count" != "$expected" ]]; then
fi
# --- 3. Every published surface must claim exactly that number. ---
# The #2729 messaging refresh dropped the count from docs/index.html,
# docs/llms.txt, pkg/npm/README.md, scripts/package-release.sh and server.json;
# section 4 below still fails if any of them states a count again unregistered.
SURFACES=(
README.md
docs/index.html
docs/llms.txt
pkg/npm/README.md
pkg/chocolatey/codebase-memory-mcp.nuspec
scripts/package-release.sh
server.json
)
# Files that legitimately say "<N> languages" about something OTHER than the
+168 -57
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env bash
# The code-scanning gate must FAIL CLOSED.
# The code-scanning gate must FAIL CLOSED -- and CodeQL must run where it gates.
#
# History. `codeql-gate` in .github/workflows/_security.yml counted open alerts
# with:
# History. `codeql-gate` in .github/workflows/_security.yml (since moved into
# .github/workflows/codeql.yml as the analyze job's alert check) counted open
# alerts with:
#
# ALERTS=$(gh api '.../code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0")
#
@@ -21,6 +22,18 @@
# A security gate that cannot read its input must stop the build, not wave it
# through. This test pins both halves of that: the permission that makes the
# read possible, and the fail-closed handling for when it still is not.
#
# It also pins where CodeQL runs. The former PR gate polled up to 150 min per
# PR for a separate CodeQL run and starved PR CI of runners; now:
# - no pull_request trigger and no job that polls for a CodeQL run;
# - PR CI calls codeql.yml as its own job (needed by ci-ok), one leg per
# selected codeql-* lane; each leg analyses its language's whole
# first-party scope (never narrowed by the diff); c-cpp compiles the
# vendored objects before tracing, then traces the first-party build plus
# the vendored files the PR changes;
# - push to main, the dry run and the release run the full scan, and the
# release's verify needs it;
# - superseded push and PR runs are cancelled, dispatched ones never.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -28,6 +41,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd)"
python3 - "$ROOT" <<'PY'
from __future__ import annotations
import json
import pathlib
import re
import sys
@@ -41,25 +55,48 @@ def require(condition: bool, message: str) -> None:
failures.append(message)
workflow = (root / ".github" / "workflows" / "_security.yml").read_text(encoding="utf-8")
workflow_dir = root / ".github" / "workflows"
# Isolate the codeql-gate job so a permission granted to some OTHER job cannot
# satisfy this test.
gate = re.search(r"\n codeql-gate:\n(?P<body>(?: .*\n|\n)*)", workflow)
require(gate is not None, "_security.yml no longer defines a codeql-gate job")
body = gate.group("body") if gate else ""
def text(name: str) -> str:
return (workflow_dir / name).read_text(encoding="utf-8")
def job_body(source: str, job: str) -> str | None:
m = re.search(rf"\n {re.escape(job)}:\n(?P<body>(?: .*\n|\n)*)", source)
return m.group("body") if m else None
def code_only(source: str) -> str:
return "\n".join(line for line in source.splitlines() if not line.lstrip().startswith("#"))
def steps(body: str) -> list[tuple[str, str]]:
"""(name, step text) per `- name:`/`- uses:` step of a job body."""
found = []
for m in re.finditer(r"(?ms)^ - (?P<step>.*?)(?=^ - |\Z)", body):
step = m.group("step")
name = re.search(r"(?m)^(?:name: |\s+name: )(.*)$", step)
found.append((name.group(1).strip() if name else step.splitlines()[0], step))
return found
workflow = text("codeql.yml")
gate = job_body(workflow, "analyze")
require(gate is not None, "codeql.yml no longer defines an analyze job")
body = gate or ""
# 1. The read has to be possible at all.
require(
re.search(r"^ permissions:\s*$", body, re.M) is not None,
"codeql-gate must declare its own permissions block -- inheriting the "
"analyze must declare its own permissions block -- inheriting the "
"workflow default leaves it without security-events access and the alert "
"API answers 403",
)
require(
re.search(r"^ security-events:\s*read\s*$", body, re.M) is not None,
"codeql-gate must request security-events: read, or the code-scanning "
"alert query 403s and the gate cannot see any alert",
re.search(r"^ security-events:\s*(read|write)\s*$", body, re.M) is not None,
"analyze must hold security-events (write implies read), or the "
"code-scanning alert query 403s and the gate cannot see any alert",
)
# 2. The failure direction. `|| echo "0"` on the alert query is the specific
@@ -71,11 +108,11 @@ require(
)
require(
"BLOCKED: cannot read code scanning alerts" in body,
"codeql-gate must fail with a named error when the alert API cannot be read",
"the alert check must fail with a named error when the alert API cannot be read",
)
require(
re.search(r"alert count was not a number", body) is not None,
"codeql-gate must reject a non-numeric alert count instead of comparing it",
"the alert check must reject a non-numeric alert count instead of comparing it",
)
# 3. The count is produced in a command substitution, so a bare `exit` inside
@@ -92,53 +129,127 @@ require(
"inside a command substitution does not stop the caller on its own",
)
# 4. Every caller must GRANT what the job requests. A reusable workflow cannot
# request more permission than its caller holds: asking for
# security-events:read from a caller that grants only contents:read does not
# downgrade, it fails the entire run with a startup_failure before a single
# job executes. Checking only the called workflow is how this test passed
# while the PR pipeline never started.
# The caller list is DERIVED, never hardcoded. An earlier revision listed
# ("pr.yml", "release.yml") by hand and stayed green while dry-run.yml -- a
# third caller nobody had listed -- failed every run at startup. A test that
# only checks the callers you remembered cannot catch the one you forgot.
workflow_dir = root / ".github" / "workflows"
# 4. Triggers: never per pull request (PR CI calls the workflow instead), on
# push to main, callable by PR CI, the dry run and the release.
on = re.search(r"(?ms)^on:\n(?P<body>.*?)(?=^\S)", workflow)
triggers = set(re.findall(r"(?m)^ ([a-z_]+):", on.group("body"))) if on else set()
require("pull_request" not in triggers and "pull_request_target" not in triggers,
"codeql.yml must not trigger on pull requests (pr.yml calls it as a job)")
require({"push", "workflow_call"} <= triggers,
f"codeql.yml must run on push to main and be callable, has {sorted(triggers)}")
require(re.search(r"(?m)^ push:\n branches: \[main\]$", workflow) is not None,
"codeql.yml's push trigger must be main")
# 5. Nothing waits for a separate CodeQL run, and _security.yml has no CodeQL.
for path in sorted(workflow_dir.glob("*.yml")):
require("workflows/codeql.yml/runs" not in code_only(path.read_text(encoding="utf-8")),
f"{path.name} polls for a CodeQL run -- CodeQL runs as a job of the caller")
require("codeql" not in code_only(text("_security.yml")).lower(),
"_security.yml must not run or wait for CodeQL")
# 6. Superseded runs are cancelled (push to main, a PR), dispatched never;
# legs of one run never cancel each other.
concurrency = re.search(r"(?ms)^ concurrency:\n(.*?)(?=^ \S)", body)
conc = concurrency.group(1) if concurrency else ""
require("matrix.language" in conc, "the concurrency group must be per language")
require("github.event_name == 'push' && github.ref" in conc,
"push runs must share a ref-scoped group (only the newest main commit is scanned)")
require("github.event.pull_request.number" in conc,
"PR scans must share a per-PR group")
require("cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}" in conc,
"only push and PR scans may be cancelled -- a dry run or release tests one commit")
# 7. The PR scan: analysis never narrowed by the diff; c-cpp prebuilds the
# vendored objects untraced and traces first-party + changed vendored.
names = [name for name, _ in steps(body)]
by_name = dict(steps(body))
def index(fragment: str) -> int:
hits = [i for i, n in enumerate(names) if fragment in n]
return hits[0] if hits else -1
init, analyze = index("Initialize CodeQL"), index("Perform CodeQL Analysis")
prebuild, retrace = index("Build third-party objects"), index("Vendored files this PR changes")
light_build, full_build = index("PR scan, first-party"), index("Build for CodeQL analysis (full)")
require(min(init, analyze, prebuild, retrace, light_build, full_build) >= 0,
f"codeql.yml analyze lost a step, has {names}")
require(-1 < prebuild < init and -1 < retrace < init,
"the vendored prebuild and the retrace list must run BEFORE Initialize CodeQL (untraced)")
require(init < light_build < analyze and init < full_build < analyze,
"the traced builds must run between Initialize CodeQL and the analysis")
for name in names:
if "Initialize CodeQL" in name or "Perform CodeQL Analysis" in name or "open code scanning alerts" in name:
require(not re.search(r"(?m)^\s+if:", by_name[name]),
f"step '{name}' must never be skipped -- a selected leg always analyses")
light_step = by_name.get(names[light_build], "") if light_build >= 0 else ""
require("scripts/build.sh --reuse-vendored --retrace-from" in light_step,
"the PR c-cpp build must reuse the untraced vendored objects and retrace the changed ones")
require("scripts/ci/codeql-retrace-vendored.sh" in (by_name.get(names[retrace], "") if retrace >= 0 else ""),
"the retrace list must come from scripts/ci/codeql-retrace-vendored.sh")
require("scripts/build.sh --vendored-only" in (by_name.get(names[prebuild], "") if prebuild >= 0 else ""),
"the PR c-cpp prebuild must be scripts/build.sh --vendored-only")
require(re.search(r"(?m)^\s+run: scripts/build\.sh\s*$", by_name.get(names[full_build], "") if full_build >= 0 else "") is not None,
"the full c-cpp scan must trace the whole production build (vendored C included)")
require(re.search(r'(?m)^ CBM_NO_CCACHE: "1"$', body) is not None,
"the CodeQL job must disable the compiler cache -- a cache hit compiles nothing to trace")
require("config-file: ./.github/codeql/codeql-config.yml" in body,
"every scan must use the first-party scope in .github/codeql/codeql-config.yml")
# 8. One configuration per language: the full matrix and PR CI's legs agree.
default = re.search(r"fromJSON\(inputs\.matrix \|\| '(?P<json>\{.*?\})'\)", body)
pr = text("pr.yml")
pr_legs = re.search(r"CODEQL='(?P<json>\[.*?\])'", pr)
full_legs = json.loads(default.group("json"))["include"] if default else []
require({leg.get("language") for leg in full_legs} ==
{"c-cpp", "javascript-typescript", "python", "actions"},
"the full scan must cover c-cpp, javascript-typescript, python and actions")
require(pr_legs is not None and json.loads(pr_legs.group("json")) == full_legs,
"pr.yml's CodeQL legs must equal codeql.yml's full matrix (one configuration per language)")
# 9. Every caller GRANTS what codeql.yml requests (a called workflow cannot
# request more than its caller holds: startup failure, not a downgrade).
# The caller list is DERIVED, never hardcoded: an earlier revision listed
# callers by hand and stayed green while a third caller nobody had listed
# failed every run at startup.
callers = sorted(
path.name
for path in workflow_dir.glob("*.yml")
if "uses: ./.github/workflows/_security.yml" in path.read_text(encoding="utf-8")
if "uses: ./.github/workflows/codeql.yml" in path.read_text(encoding="utf-8")
)
require(
len(callers) >= 3,
"expected at least 3 callers of _security.yml, found: "
+ (", ".join(callers) or "none")
+ " -- if a caller was deliberately removed, lower this floor on purpose",
)
require({"pr.yml", "dry-run.yml", "release.yml"} <= set(callers),
"PR CI, the dry run and the release must call codeql.yml, callers: "
+ (", ".join(callers) or "none"))
for caller_name in callers:
caller = (workflow_dir / caller_name).read_text(encoding="utf-8")
# The next sibling may be a comment line, not a key, so the lookahead has to
# accept any 2-space-indented non-space -- matching only ` \w` silently
# failed to find release.yml's job at all.
call = re.search(
r"\n security:\n(?P<body>(?: .*\n|\n)*?)(?=\n? \S|\Z)", caller
)
require(call is not None, f"{caller_name} no longer has a security: job")
call_body = call.group("body") if call else ""
require(
"uses: ./.github/workflows/_security.yml" in call_body,
f"{caller_name}'s security job must call _security.yml",
)
require(
re.search(r"^ security-events:\s*read\s*$", call_body, re.M) is not None,
f"{caller_name} must grant security-events: read to the security job -- "
"a called workflow cannot request more than its caller grants, and the "
"run fails at startup rather than degrading",
)
require(
re.search(r"^ actions:\s*read\s*$", call_body, re.M) is not None,
f"{caller_name} must grant actions: read to the security job",
)
call_body = job_body(text(caller_name), "codeql") or ""
require("uses: ./.github/workflows/codeql.yml" in call_body,
f"{caller_name} must call codeql.yml from a job named codeql")
for perm, level in (("security-events", "write"), ("actions", "read"), ("contents", "read")):
require(re.search(rf"^ {perm}:\s*{level}\s*$", call_body, re.M) is not None,
f"{caller_name} must grant {perm}: {level} to the codeql job")
light = re.search(r"(?m)^ light: true$", call_body) is not None
require(light == (caller_name == "pr.yml"),
f"{caller_name}: only PR CI runs the PR scan (light: true); "
"the dry run and the release run the full scan")
# 10. PR CI: the codeql job runs for any selected codeql-* lane and ci-ok
# needs it; the release publishes nothing without the CodeQL pass.
pr_codeql = job_body(pr, "codeql") or ""
cond = re.search(r"(?m)^ if: (.*)$", pr_codeql)
for lane in ("codeql-cpp", "codeql-js", "codeql-py", "codeql-actions"):
require(cond is not None and f"'{lane}'" in cond.group(1),
f"pr.yml codeql must run when lane {lane} is selected")
require("matrix: ${{ needs.changes.outputs.codeql }}" in pr_codeql,
"pr.yml codeql must run the legs the changes job selected")
ci_ok = job_body(pr, "ci-ok") or ""
require(re.search(r"(?m)^ needs: \[[^\]]*\bcodeql\b[^\]]*\]$", ci_ok) is not None,
"ci-ok must need the codeql job")
verify = job_body(text("release.yml"), "verify") or ""
require(re.search(r"(?m)^ needs: \[[^\]]*\bcodeql\b[^\]]*\]$", verify) is not None,
"release.yml verify must need the codeql job")
require("needs.codeql.result == 'success'" in verify,
"release.yml verify must require needs.codeql.result == 'success'")
if failures:
print("Security gate fail-closed contract FAILED:")
+111 -1
View File
@@ -35,6 +35,7 @@ trap 'rm -rf "$WORK"' EXIT
# explicit path): a bare "bash" argv can resolve to the WSL alias on Windows.
python3 - "$BASH" "$SELECTOR" "$CASES" "$WORK" <<'PY'
import json
import os
import pathlib
import random
import subprocess
@@ -47,12 +48,14 @@ failures = []
# Every lane of the full PR pipeline, spelled out here rather than read from
# the selector, so a lane silently dropped from the selector's universe fails.
FULL = {
"codeql-gate", "diag", "license-gate", "lint", "lint-mem", "lsan-macos",
"diag", "license-gate", "lint", "lint-mem", "lsan-macos",
"memwaste", "msan", "pkg-wrappers", "security-static",
"shard-completeness", "smoke-mac", "smoke-ubuntu", "smoke-win",
"tsan-arm", "tsan-mac", "tsan-x86", "unix-arm64", "unix-macos-intel",
"unix-macos14", "unix-x86", "windows", "windows-guards",
"codeql-cpp", "codeql-js", "codeql-py", "codeql-actions",
}
CODEQL = {"codeql-cpp", "codeql-js", "codeql-py", "codeql-actions"}
UNIVERSE = FULL | {"contracts"}
@@ -201,6 +204,113 @@ for bad in ("many", ""): # "" = a workflow whose count went missing
failures.append(f"--expect-files {bad!r} must be a usage error (exit 2), never "
f"a silently skipped truncation check")
# 8. The CodeQL PR scans (codeql-* lanes) fail closed:
# (a) a C/C++ change -- first-party or vendored -- always selects the C scan;
# (b) an undeterminable diff (an empty or truncated list here; a diff the
# retrace helper cannot compute below) selects every language's full
# first-party scan, and so does a change to the workflows, the build or
# the CodeQL machinery;
# (c) a vendored-only change scans the changed vendored files: the C lane
# is selected and the retrace helper lists exactly those files, which
# codeql.yml then compiles under tracing.
# Tests, docs and vendored non-C select no CodeQL lane.
def lanes_for(files, args=()):
rc, out, _ = run(list(args), "".join(f + "\n" for f in files))
return set(json.loads(out)["lanes"]) if rc == 0 else None
for c_file in ("src/pipeline/pass_calls.c", "src/foundation/mem_core.c",
"internal/cbm/extract_calls.c", "internal/cbm/lsp/go_lsp.c",
"internal/cbm/cbm.h", "src/newdir/x.c",
"internal/cbm/vendored/grammars/go/parser.c",
"internal/cbm/vendored/lz4/lz4.c", "vendored/sqlite3/sqlite3.c",
"vendored/mimalloc/src/options.c"):
got = lanes_for([c_file])
if got is None or "codeql-cpp" not in got:
failures.append(f"codeql (a): a C/C++ change ({c_file}) must select codeql-cpp, "
f"got {sorted(got or [])}")
for label, files, args in (("empty file list", [], ()),
("truncated list", ["docs/a.md"], ("--expect-files", "5")),
("a workflow", [".github/workflows/stale.yml"], ()),
("Makefile.cbm", ["Makefile.cbm"], ()),
("scripts/build.sh", ["scripts/build.sh"], ()),
("the selector", ["scripts/ci/select-lanes.sh"], ()),
("the CodeQL config", [".github/codeql/codeql-config.yml"], ()),
("the retrace helper", ["scripts/ci/codeql-retrace-vendored.sh"], ())):
got = lanes_for(files, args)
if got is None or not CODEQL <= got:
failures.append(f"codeql (b): {label} must select every CodeQL language, "
f"got {sorted((got or set()) & CODEQL)}")
for quiet in ("tests/test_store.c", "tests/fixtures/x.py", "README.md", "docs/a.md",
"graph-ui/vendor/lib.js", "scripts/repro.sh"):
got = lanes_for([quiet])
if got is None or got & CODEQL:
failures.append(f"codeql: {quiet} must select no CodeQL lane, "
f"got {sorted((got or set()) & CODEQL)}")
for language, path in (("javascript-typescript", "graph-ui/src/App.tsx"),
("javascript-typescript", "pkg/npm/install.js"),
("python", "scripts/test-impact/x.py"),
("actions", ".github/workflows/pr.yml")):
rc, out, _ = run(["--codeql-scope", language], path + "\n")
if rc != 0 or out.split() != [path]:
failures.append(f"--codeql-scope {language} must keep {path}, printed {out!r}")
rc, _, err = run(["--codeql-scope", "go"], "src/main.c\n")
if rc != 2 or "Please consult --help." not in err:
failures.append("--codeql-scope with an unknown language must be a usage error (exit 2)")
retrace = pathlib.Path(selector).parent / "codeql-retrace-vendored.sh"
env = {k: v for k, v in os.environ.items() if k != "GITHUB_EVENT_NAME"}
changed = work / "codeql-changed.txt"
changed.write_text("src/main.c\ninternal/cbm/vendored/grammars/go/parser.c\n"
"vendored/sqlite3/sqlite3.c\nvendored/no-such-file.c\n"
"internal/cbm/grammar_json.c\ninternal/cbm/ts_runtime.c\n"
"graph-ui/src/App.tsx\n", encoding="utf-8")
listed = work / "codeql-retrace.txt"
proc = subprocess.run([bash, str(retrace), "--changed", str(changed), str(listed)],
capture_output=True, text=True, env=env)
got = listed.read_text(encoding="utf-8").split() if listed.exists() else None
if proc.returncode != 0 or got != ["internal/cbm/grammar_go.c",
"internal/cbm/grammar_json.c",
"internal/cbm/ts_runtime.c",
"vendored/sqlite3/sqlite3.c"]:
failures.append(f"codeql (c): the retrace helper must list the translation units for "
f"changed vendored code and wrappers, got {got} (exit {proc.returncode})")
# Native Windows Python emits CRLF even when the change list uses LF. Pin
# that producer/consumer boundary on every platform, including direct wrapper
# paths that must survive verbatim rather than being mapped to a family name.
crlf_root = work / "crlf-selector"
crlf_scripts = crlf_root / "scripts" / "ci"
crlf_scripts.mkdir(parents=True)
(crlf_scripts / retrace.name).write_bytes(retrace.read_bytes())
expected_retrace = ["internal/cbm/grammar_go.c", "internal/cbm/grammar_json.c",
"internal/cbm/ts_runtime.c", "vendored/sqlite3/sqlite3.c"]
for path in expected_retrace:
fixture = crlf_root / path
fixture.parent.mkdir(parents=True, exist_ok=True)
fixture.touch()
(crlf_scripts / "select-lanes.sh").write_bytes(
b"#!/usr/bin/env bash\nprintf '%s\\r\\n' "
b"internal/cbm/vendored/grammars/go/parser.c internal/cbm/grammar_json.c "
b"internal/cbm/ts_runtime.c vendored/sqlite3/sqlite3.c\n")
proc = subprocess.run([bash, str(crlf_scripts / retrace.name),
"--changed", str(changed), str(listed)],
capture_output=True, text=True, env=env)
got = listed.read_bytes() if listed.exists() else None
if proc.returncode != 0 or got != ("\n".join(expected_retrace) + "\n").encode("utf-8"):
failures.append(f"codeql CRLF: the retrace helper must preserve every changed "
f"wrapper and write LF entrypoints, got {got!r} (exit {proc.returncode})")
for label, args in (("an unreadable change list", ["--changed", str(work / "missing.txt")]),
("no pull request", [])):
listed.write_text("stale\n", encoding="utf-8")
proc = subprocess.run([bash, str(retrace), *args, str(listed)],
capture_output=True, text=True, env=env)
if proc.returncode != 0 or listed.read_text(encoding="utf-8") != "" \
or "undetermined" not in proc.stdout:
failures.append(f"codeql (b): with {label} the retrace helper must succeed with an "
f"empty list (the scan still covers all first-party C), "
f"exit {proc.returncode}: {proc.stdout.strip()!r}")
if failures:
print("LANE SELECTOR CONTRACT VIOLATED:")
for failure in failures:
+2
View File
@@ -386,6 +386,7 @@ if [ -d "$ROOT/scripts" ] && [ -f "$ROOT/scripts/test.sh" ]; then
HELP_ENTRIES="
scripts/test.sh
scripts/build.sh
scripts/ci/codeql-retrace-vendored.sh
scripts/lint.sh
scripts/smoke-local.sh
scripts/soak-legs.sh
@@ -433,6 +434,7 @@ test-infrastructure/vm/win.sh
STRICT_ENTRIES="
scripts/test.sh
scripts/build.sh
scripts/ci/codeql-retrace-vendored.sh
scripts/lint.sh
scripts/smoke-local.sh
scripts/soak-legs.sh
+38 -4
View File
@@ -1103,8 +1103,8 @@ TEST(watcher_stop_and_unwatch_cancel_blocked_git_without_backstop) {
watcher_windows_image_identity_t expected_git;
ASSERT_TRUE(watcher_windows_capture_image_identity(fake_git, &expected_git));
const char *old_path_raw = getenv("PATH");
const char *old_marker_raw = getenv(WATCHER_TEST_BLOCKING_GIT_MARKER_ENV);
const char *old_path_raw = cbm_runtime_getenv("PATH");
const char *old_marker_raw = cbm_runtime_getenv(WATCHER_TEST_BLOCKING_GIT_MARKER_ENV);
char *old_path = old_path_raw ? cbm_strdup(old_path_raw) : NULL;
char *old_marker = old_marker_raw ? cbm_strdup(old_marker_raw) : NULL;
size_t path_size = strlen(bin) + 2 + (old_path ? strlen(old_path) : 0);
@@ -1238,8 +1238,8 @@ TEST(watcher_stop_and_unwatch_cancel_blocked_git_without_backstop) {
ASSERT_TRUE(watcher_test_self_image(self));
ASSERT_EQ(symlink(self, fake_git), 0);
const char *old_path_raw = getenv("PATH");
const char *old_marker_raw = getenv(WATCHER_TEST_BLOCKING_GIT_MARKER_ENV);
const char *old_path_raw = cbm_runtime_getenv("PATH");
const char *old_marker_raw = cbm_runtime_getenv(WATCHER_TEST_BLOCKING_GIT_MARKER_ENV);
char *old_path = old_path_raw ? strdup(old_path_raw) : NULL;
char *old_marker = old_marker_raw ? strdup(old_marker_raw) : NULL;
size_t path_size = strlen(bin) + 2 + (old_path ? strlen(old_path) : 0);
@@ -3995,7 +3995,41 @@ TEST(watcher_null_watch_count) {
* SUITE
* ══════════════════════════════════════════════════════════════════ */
TEST(watcher_unicode_root_is_not_pruned_as_missing) {
char *parent = th_mktempdir("cbm_watch_utf8");
ASSERT_NOT_NULL(parent);
char root[CBM_SZ_4K];
snprintf(root, sizeof(root), "%s/\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E", parent);
ASSERT_EQ(th_mkdir_p(root), 0);
const char *saved = cbm_runtime_getenv("CBM_WATCHER_PRUNE_GRACE_S");
char *saved_copy = saved ? strdup(saved) : NULL;
cbm_setenv("CBM_WATCHER_PRUNE_GRACE_S", "0", 1);
cbm_store_t *store = cbm_store_open_memory();
cbm_store_upsert_project(store, "unicode-root", root);
cbm_watcher_t *watcher = cbm_watcher_new(store, NULL, NULL);
cbm_watcher_watch(watcher, "unicode-root", root);
for (int i = 0; i < 8; i++) {
cbm_watcher_poll_once(watcher);
}
cbm_project_t info = {0};
bool kept = cbm_watcher_watch_count(watcher) == 1 &&
cbm_store_get_project(store, "unicode-root", &info) == CBM_STORE_OK;
cbm_project_free_fields(&info);
cbm_watcher_free(watcher);
cbm_store_close(store);
if (saved_copy) {
cbm_setenv("CBM_WATCHER_PRUNE_GRACE_S", saved_copy, 1);
} else {
cbm_unsetenv("CBM_WATCHER_PRUNE_GRACE_S");
}
free(saved_copy);
ASSERT_EQ(th_rmtree(parent), 0);
ASSERT_TRUE(kept);
PASS();
}
SUITE(watcher) {
RUN_TEST(watcher_unicode_root_is_not_pruned_as_missing);
/* Adaptive interval */
RUN_TEST(poll_interval_base);
RUN_TEST(poll_interval_scaling);