diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 000000000..5c2a1ea68 --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,27 @@ +# CodeQL scope: the repository's first-party source. +# +# Read by .github/workflows/codeql.yml (every scan, full and PR) and by +# scripts/ci/select-lanes.sh (which changed paths select which language's PR +# scan). Keep the lists plain: one path prefix per line, no globs. +# +# paths / paths-ignore narrow what the INTERPRETED languages extract +# (javascript-typescript, python, actions). For c-cpp they have no effect: +# CodeQL extracts exactly what the traced build compiles. A PR's c-cpp scan +# keeps vendored C out by compiling it before tracing starts (except vendored +# files the PR changes); the full scan on main, the dry run and the release +# traces everything, vendored C included. +name: cbm first-party source +paths: + - src + - internal + - graph-ui + - pkg + - scripts + - tests + - .github +paths-ignore: + - internal/cbm/vendored + - graph-ui/vendor + - graph-ui/fixtures + - graph-ui/verification + - tests/fixtures diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml index 6fcda15e4..741e3c438 100644 --- a/.github/workflows/_lint.yml +++ b/.github/workflows/_lint.yml @@ -1,5 +1,5 @@ # Reusable: lint only (cppcheck + clang-format). -# Security-static and CodeQL gate are separate — see _security.yml. +# Security-static is separate — see _security.yml; CodeQL — see codeql.yml. name: Lint & Security on: diff --git a/.github/workflows/_security.yml b/.github/workflows/_security.yml index 666573471..40514091a 100644 --- a/.github/workflows/_security.yml +++ b/.github/workflows/_security.yml @@ -1,4 +1,6 @@ -# Reusable: security-static + CodeQL gate. +# Reusable: security-static + license gate. +# CodeQL is not here: PRs do not run it. It runs on push to main, weekly, and +# inside dry-run.yml / release.yml, which call codeql.yml directly. # Runs independently from lint/test/build — does not block the main pipeline. # Affects overall workflow success status. name: Security Gate @@ -52,96 +54,3 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: scripts/audit-license-provenance.py - codeql-gate: - if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'codeql-gate') }} - runs-on: ubuntu-latest - timeout-minutes: 240 - # Without security-events:read this job inherits the workflow's - # contents:read, the code-scanning alert API answers 403, and the gate - # below reported "0 alerts" on every run it has ever made. actions:read is - # what the CodeQL-run poll above needs for the same reason. - permissions: - contents: read - security-events: read - actions: read - steps: - - name: Wait for CodeQL on current commit (max 150 min) - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # On pull_request events github.sha is the synthetic merge commit; - # CodeQL runs are recorded against the PR head SHA. - CURRENT_SHA="${{ github.event.pull_request.head.sha || github.sha }}" - echo "Waiting for CodeQL to complete on $CURRENT_SHA..." - # 300 attempts x 30s = 150 min. The previous budget was 90 x 30s = 45 min, - # which is shorter than CodeQL actually takes on this repository: a measured - # run on PR #1426 (head 7b72652a) completed with conclusion=success after - # 124 min, 7 minutes AFTER this gate had already given up. That marked at - # least five contributor PRs red for a scan that passed. The job's own - # timeout-minutes is 240, so 150 still leaves headroom. - for attempt in $(seq 1 300); do - LATEST=$(gh api "repos/${{ github.repository }}/actions/workflows/codeql.yml/runs?head_sha=$CURRENT_SHA&per_page=1" \ - --jq '.workflow_runs[] | "\(.conclusion) \(.status)"' 2>/dev/null | head -1 || echo "") - if [ -z "$LATEST" ]; then - echo " $attempt/300: no run yet..."; sleep 30; continue - fi - CONCLUSION=$(echo "$LATEST" | cut -d' ' -f1) - STATUS=$(echo "$LATEST" | cut -d' ' -f2) - if [ "$STATUS" = "completed" ] && [ "$CONCLUSION" = "success" ]; then - echo "=== CodeQL passed ==="; exit 0 - elif [ "$STATUS" = "completed" ]; then - echo "BLOCKED: CodeQL $CONCLUSION"; exit 1 - fi - echo " $attempt/300: $STATUS..."; sleep 30 - done - echo "BLOCKED: CodeQL timeout"; exit 1 - - - name: Check for open code scanning alerts - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, - # which looks defensive and is the opposite. gh writes the API error - # BODY to stdout, so a 403 made ALERTS the string - # {"message":"Resource not accessible by integration",...,"status":"403"}0 - # every `[` comparison then failed as non-integer, the `if` took its - # false branch, and the step printed "CodeQL gate passed (0 alerts)" - # and exited 0. That is exactly how a security gate should NEVER fail: - # silently, in the direction of "everything is fine". An unreadable - # alert list is now an error, not a zero. - # NOTE: this runs in a command substitution, so it must `return`, not - # `exit` -- an `exit` here would only end the subshell and let the - # caller continue with an empty count, recreating the very bug this - # replaces. The callers below check the status explicitly. - count_open_alerts() { - local body status - body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ - --jq 'length' 2>/dev/null) - status=$? - if [ $status -ne 0 ]; then - echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 - echo " Refusing to treat an unreadable alert list as zero." >&2 - echo " Most likely cause: the job lacks security-events:read." >&2 - return 1 - fi - case "$body" in - ''|*[!0-9]*) - echo "BLOCKED: alert count was not a number: ${body:-}" >&2 - return 1 - ;; - esac - printf '%s' "$body" - } - echo "Waiting 60s for alert API to settle..." - sleep 60 - ALERTS=$(count_open_alerts) || exit 1 - sleep 15 - ALERTS2=$(count_open_alerts) || exit 1 - [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 - if [ "$ALERTS" -gt 0 ]; then - echo "BLOCKED: $ALERTS open alert(s)" - gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ - --jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true - exit 1 - fi - echo "=== CodeQL gate passed (0 alerts) ===" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 621d407fc..f5cfb59d6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,47 +1,194 @@ name: CodeQL SAST +# One analysis per language: c-cpp, javascript-typescript (graph-ui, pkg/npm), +# python, actions. Scope: .github/codeql/codeql-config.yml (first-party source). +# +# FULL scan -- push to main, weekly, dry run, release: every language; c-cpp +# traces the whole production build, vendored C included. +# PR scan -- pr.yml's `codeql` job (light: true), needed by ci-ok. One leg per +# language whose scope the PR changes (codeql-* lanes of +# scripts/ci/select-lanes.sh; an undeterminable diff or a change to the +# workflows, the build or the CodeQL machinery selects every language). +# Each leg analyses its language's WHOLE first-party scope, so cross-file +# data flow is kept; c-cpp compiles the vendored objects before tracing +# starts, except vendored files the PR changes. Diff-informed analysis then +# restricts the REPORTED alerts to the changed lines; what is extracted and +# analysed is never narrowed by the diff. +# There is no pull_request trigger and no job anywhere that polls for a CodeQL +# run: the former PR gate waited up to 150 min per PR for a full analysis and, +# with ~30 PRs pushed at once, starved PR CI for hours. on: push: branches: [main] - pull_request: - branches: [main] + schedule: + - cron: "23 5 * * 1" + workflow_dispatch: + inputs: + light: + description: "PR-style c-cpp build (vendored objects compiled untraced) -- to time it" + type: boolean + default: false + workflow_call: + inputs: + light: + description: "PR scan: vendored C compiled untraced except changed files; diff-informed results" + type: boolean + default: false + matrix: + description: 'The legs to run, {"include":[...]} (pr.yml: the selected codeql-* lanes); empty = every language' + type: string + default: "" permissions: contents: read -# One analysis per pull request and one per pushed branch. A newer push to main -# cancels the analysis of an older main commit; only the newest commit's -# results matter for main, and a merge-heavy day otherwise queues one 40-100 -# minute analysis per merge ahead of every pull request run. -concurrency: - group: codeql-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: analyze: runs-on: ubuntu-latest - # CodeQL needs security-events: write to upload results; scoped to this job so - # the workflow's top-level token stays read-only (Scorecard TokenPermissions). + strategy: + fail-fast: false + # The same legs (and keys) everywhere, so a PR's analysis and main's + # share one code-scanning configuration per language. + matrix: ${{ fromJSON(inputs.matrix || '{"include":[{"language":"c-cpp","build-mode":"manual","lane":"codeql-cpp"},{"language":"javascript-typescript","build-mode":"none","lane":"codeql-js"},{"language":"python","build-mode":"none","lane":"codeql-py"},{"language":"actions","build-mode":"none","lane":"codeql-actions"}]}') }} + # Superseded runs are always cancelled: a newer push to main cancels the + # analysis of an older main commit (batched merges -> only the newest main + # commit is scanned), a newer push to a PR cancels that PR's scan. Every + # other trigger gets a group of its own and is never cancelled: a dry run + # or a release tests ONE commit, and two dispatched together must not + # cancel each other's gate. Per language, so legs never cancel each other. + # In a called workflow `github` is the CALLER's context: event_name is + # pull_request for pr.yml, github.workflow names the dry run or release. + concurrency: + group: codeql-${{ matrix.language }}-${{ github.event_name == 'push' && github.ref || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + # CodeQL needs security-events: write to upload results (and read for the + # alert check); scoped to this job so the workflow's top-level token stays + # read-only (Scorecard TokenPermissions). permissions: security-events: write actions: read contents: read + env: + # One code-scanning configuration for every trigger. The action derives + # the key from the RUN's workflow file, which for a called workflow is + # the caller: pr.yml / dry-run.yml / release.yml would each open a + # configuration of their own, and on main an alert fixed by a later push + # would stay open in a stale one. The value is the key the push-to-main + # analysis has always used (the action documents this variable as + # settable for non-standard setups). + CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql.yml:analyze + # Diff-informed analysis: codeql-action v4.38.1 (CLI 2.27.0) reports a + # pull request's alerts only on its changed lines when the + # diff_informed_queries feature is on (default on; this variable forces + # it, src/feature-flags.ts). It applies only when the event payload is a + # pull request -- pr.yml's call -- and is inert on main, dry run, release. + CODEQL_ACTION_DIFF_INFORMED_QUERIES: ${{ inputs.light && 'true' || '' }} + # No compiler cache: a cache hit skips the compiler, and CodeQL extracts + # only what a traced compiler invocation actually compiles. + CBM_NO_CCACHE: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The PR scan reads the merge commit's first parent (the base) to + # find the vendored files the PR changes. + fetch-depth: ${{ inputs.light && 2 || 1 }} - name: Install build dependencies + if: ${{ matrix.language == 'c-cpp' }} run: sudo apt-get update && sudo apt-get install -y zlib1g-dev + # PR c-cpp: compile the third-party objects (tree-sitter grammars, + # sqlite, mimalloc, lz4, zstd, the tree-sitter runtime: ~1.3 GB of + # vendored C against ~45 MB first-party) BEFORE tracing starts, so the + # traced build below compiles first-party code only ... + - name: Build third-party objects (PR scan, untraced) + if: ${{ matrix.language == 'c-cpp' && inputs.light }} + run: scripts/build.sh --vendored-only + + # ... except the vendored files this PR changes: those are rebuilt + # under tracing (an undeterminable diff leaves the list empty; the scan + # still covers all first-party C). + - name: Vendored files this PR changes (traced too) + if: ${{ matrix.language == 'c-cpp' && inputs.light }} + run: scripts/ci/codeql-retrace-vendored.sh "$RUNNER_TEMP/codeql-retrace.txt" + - name: Initialize CodeQL uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: - languages: c-cpp - build-mode: manual + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + config-file: ./.github/codeql/codeql-config.yml - - name: Build for CodeQL analysis + - name: Build for CodeQL analysis (full) + if: ${{ matrix.language == 'c-cpp' && !inputs.light }} run: scripts/build.sh + - name: Build for CodeQL analysis (PR scan, first-party + changed vendored) + if: ${{ matrix.language == 'c-cpp' && inputs.light }} + run: scripts/build.sh --reuse-vendored --retrace-from "$RUNNER_TEMP/codeql-retrace.txt" + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: - category: "/language:c-cpp" + category: "/language:${{ matrix.language }}" + + # Alert policy: the default branch carries ZERO open code-scanning + # alerts. Any open alert on the analysed ref fails this leg: on main the + # commit turns red and GitHub notifies whoever triggered the run; the + # dry run and the release cannot pass; a PR (refs/pull//merge: what + # its changed lines introduce) fails ci-ok. With a zero baseline "any + # open alert" is exactly "a new alert"; a dismissal (with a reason, in + # the Security tab) holds on every ref. Every leg checks after its own + # upload, so each language's alerts are seen by at least its own leg. + # Moved from _security.yml's former PR-CI codeql-gate job. + - name: Check for open code scanning alerts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ALERT_REF: ${{ github.ref }} + run: | + # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, + # which looks defensive and is the opposite. gh writes the API error + # BODY to stdout, so a 403 made ALERTS the string + # {"message":"Resource not accessible by integration",...,"status":"403"}0 + # every `[` comparison then failed as non-integer, the `if` took its + # false branch, and the step printed "CodeQL gate passed (0 alerts)" + # and exited 0. That is exactly how a security gate should NEVER fail: + # silently, in the direction of "everything is fine". An unreadable + # alert list is now an error, not a zero. + # NOTE: this runs in a command substitution, so it must `return`, not + # `exit` -- an `exit` here would only end the subshell and let the + # caller continue with an empty count, recreating the very bug this + # replaces. The callers below check the status explicitly. + count_open_alerts() { + local body status + body=$(gh api "repos/${{ github.repository }}/code-scanning/alerts?state=open&ref=$ALERT_REF" \ + --jq 'length' 2>/dev/null) + status=$? + if [ $status -ne 0 ]; then + echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 + echo " Refusing to treat an unreadable alert list as zero." >&2 + echo " Most likely cause: the job lacks security-events:read." >&2 + return 1 + fi + case "$body" in + ''|*[!0-9]*) + echo "BLOCKED: alert count was not a number: ${body:-}" >&2 + return 1 + ;; + esac + printf '%s' "$body" + } + echo "Waiting 60s for alert API to settle..." + sleep 60 + ALERTS=$(count_open_alerts) || exit 1 + sleep 15 + ALERTS2=$(count_open_alerts) || exit 1 + [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 + if [ "$ALERTS" -gt 0 ]; then + echo "::error title=Open code scanning alerts::$ALERTS open alert(s) on $ALERT_REF -- fix, or dismiss with a reason in the Security tab" + echo "BLOCKED: $ALERTS open alert(s)" + gh api "repos/${{ github.repository }}/code-scanning/alerts?state=open&ref=$ALERT_REF" \ + --jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true + exit 1 + fi + echo "=== CodeQL gate passed (0 alerts) ===" diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 600dbad04..463424092 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -2,9 +2,10 @@ # Each step can be skipped for faster iteration. # # Pipeline: lint → test → build + candidate scan/select → smoke/soak -# Security: security-static + codeql-gate (independent island) +# Security: security-static + license-gate (independent island) +# CodeQL: codeql.yml full scan of this commit + alert check (independent island) # -# Security does NOT block lint/test/build/smoke/soak. All jobs must pass +# Security and CodeQL do NOT block lint/test/build/smoke/soak. All jobs must pass # for the overall workflow to be green. # # Candidate VirusTotal scanning and tuple-local selection are part of the @@ -62,15 +63,24 @@ jobs: # ── Security (independent island — does not block main pipeline) ── security: uses: ./.github/workflows/_security.yml - # The called job requests security-events:read to count code-scanning - # alerts; a reusable workflow cannot request more than its caller grants, - # so withholding this here fails the whole run at startup. + # No job in _security.yml reads code scanning or the Actions API any more + # (the CodeQL gate moved to codeql.yml), so the token stays read-only. permissions: contents: read - security-events: read - actions: read secrets: inherit + # ── CodeQL: full scan of this exact commit + the open-alert check ── + # Every language, vendored C included (PRs run the narrower PR scan). + # codeql.yml uploads results and reads the alerts: a called workflow + # cannot request more than its caller grants, so withholding either + # permission fails the whole run at startup. + codeql: + uses: ./.github/workflows/codeql.yml + permissions: + contents: read + security-events: write + actions: read + # ── Lint (cppcheck + clang-format) ──────────────────────────── lint: if: ${{ inputs.skip_lint != true }} diff --git a/.github/workflows/pr-acknowledgement.yml b/.github/workflows/pr-acknowledgement.yml index ef36a9fb0..760483abb 100644 --- a/.github/workflows/pr-acknowledgement.yml +++ b/.github/workflows/pr-acknowledgement.yml @@ -51,17 +51,25 @@ jobs: run: | set -euo pipefail MESSAGE_FILE=.github/pr-acknowledgement.md + BODY_FILE="$RUNNER_TEMP/pr-acknowledgement-body.md" if [ ! -f "$MESSAGE_FILE" ]; then echo "No $MESSAGE_FILE in the base tree — acknowledgements are off." exit 0 fi - # An empty or whitespace-only file is the documented off switch. - if [ -z "$(tr -d '[:space:]' < "$MESSAGE_FILE")" ]; then - echo "$MESSAGE_FILE is blank — acknowledgements are off." + # HTML comments in the template are notes for maintainers. Posted + # verbatim they become text a reader cannot see but a tool can, and + # our own PR screening holds every comment that carries hidden text. + # Strip them (an unterminated one runs to the end, as GitHub renders + # it) so the posted body is exactly what the contributor sees. + perl -0777 -pe 's/|\z)[ \t]*\n?//gs' "$MESSAGE_FILE" > "$BODY_FILE" + + # A body with no visible text is the documented off switch. + if [ -z "$(tr -d '[:space:]' < "$BODY_FILE")" ]; then + echo "$MESSAGE_FILE has no visible text — acknowledgements are off." exit 0 fi - gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "$MESSAGE_FILE" + gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file "$BODY_FILE" echo "Acknowledged PR #${PR_NUMBER}." diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 815146253..81a838e80 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -26,13 +26,10 @@ jobs: security: needs: [changes] uses: ./.github/workflows/_security.yml - # The called job requests security-events:read to count code-scanning - # alerts; a reusable workflow cannot request more than its caller grants, - # so withholding this here fails the whole run at startup. + # No job in _security.yml reads code scanning or the Actions API any more + # (the CodeQL gate moved to codeql.yml), so the token stays read-only. permissions: contents: read - security-events: read - actions: read secrets: inherit with: lanes: ${{ needs.changes.outputs.lanes }} @@ -128,6 +125,7 @@ jobs: full: ${{ steps.select.outputs.full }} lanes: ${{ steps.select.outputs.lanes }} smoke: ${{ steps.select.outputs.smoke }} + codeql: ${{ steps.select.outputs.codeql }} steps: - name: List the changed files env: @@ -180,6 +178,10 @@ jobs: SMOKE='[{"os":"ubuntu-latest","lane":"smoke-ubuntu"},{"os":"macos-14","lane":"smoke-mac"},{"os":"windows-latest","lane":"smoke-win"}]' LANES=$(sed -n 's/^lanes=//p' "$RUNNER_TEMP/selection.txt") echo "smoke={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$SMOKE")}" >> "$GITHUB_OUTPUT" + # The CodeQL legs, tagged the same way (codeql.yml's full matrix has + # the same entries, so PR and main share a configuration). + CODEQL='[{"language":"c-cpp","build-mode":"manual","lane":"codeql-cpp"},{"language":"javascript-typescript","build-mode":"none","lane":"codeql-js"},{"language":"python","build-mode":"none","lane":"codeql-py"},{"language":"actions","build-mode":"none","lane":"codeql-actions"}]' + echo "codeql={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$CODEQL")}" >> "$GITHUB_OUTPUT" # ── Light smoke: build the PRODUCTION binary and run the core smoke on the # three RELIABLE NATIVE platforms. Catches built-binary regressions at PR @@ -361,13 +363,34 @@ jobs: - name: Contract steps only run: scripts/test.sh --contracts-only + # ── CodeQL PR scan (lanes codeql-cpp / -js / -py / -actions): one leg per + # language whose first-party scope this PR changes; each analyses that + # language's whole first-party scope, c-cpp with the vendored objects + # compiled before tracing (except vendored files the PR changes), and + # reports alerts on the changed lines (diff-informed). A job of this run, + # needed by ci-ok -- nothing waits for a separate CodeQL run. Push to + # main, dry run and release run the full scan (codeql.yml). ── + codeql: + needs: [changes] + if: ${{ !cancelled() && needs.changes.result == 'success' && (contains(fromJSON(needs.changes.outputs.lanes), 'codeql-cpp') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-js') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-py') || contains(fromJSON(needs.changes.outputs.lanes), 'codeql-actions')) }} + uses: ./.github/workflows/codeql.yml + # codeql.yml uploads results and reads alerts: a called workflow cannot + # request more than its caller grants (startup failure otherwise). + permissions: + contents: read + security-events: write + actions: read + with: + light: true + matrix: ${{ needs.changes.outputs.codeql }} + ci-ok: # The one required context (besides dco) — fails unless every PR stage # succeeded, so matrix renames can never silently deadlock merges, and # every lane the selector chose ran green: `skipped` is OK only for a lane # nobody selected (a selected lane that skipped is a silent gate loss). # memwaste and the test-impact shadow are report-only and not needed here. - needs: [changes, security, lint, test, pr-smoke, contracts] + needs: [changes, security, lint, test, pr-smoke, contracts, codeql] if: ${{ always() }} runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 134b425e2..707c69924 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,8 @@ # Release pipeline: lint → test → build → smoke/soak → draft → verify → publish # -# Security (security-static + codeql-gate) starts immediately and runs in -# parallel with lint/test/build/smoke/soak. It only blocks the final verify +# Security (security-static + license-gate) and CodeQL (codeql.yml: full scan +# of this commit + the open-alert check) start immediately and run in +# parallel with lint/test/build/smoke/soak. They only block the final verify # step — everything else proceeds independently. name: Release @@ -44,15 +45,24 @@ jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: uses: ./.github/workflows/_security.yml - # The called job requests security-events:read to count code-scanning - # alerts; a reusable workflow cannot request more than its caller grants, - # so withholding this here fails the whole run at startup. + # No job in _security.yml reads code scanning or the Actions API any more + # (the CodeQL gate moved to codeql.yml), so the token stays read-only. permissions: contents: read - security-events: read - actions: read secrets: inherit + # ── CodeQL (starts immediately, blocks only verify) ────────────── + # Full scan of this exact commit (every language, vendored C included) + + # the open-alert check. codeql.yml uploads results and reads the alerts: + # a called workflow cannot request more than its caller grants, so + # withholding either permission fails the whole run at startup. + codeql: + uses: ./.github/workflows/codeql.yml + permissions: + contents: read + security-events: write + actions: read + # ── 0. Preflight: refuse malformed dispatch inputs ────────────── # The tag is inputs.version VERBATIM. A bare "0.10.7" publishes a release the # installers can never resolve (they fetch releases/download/v/...), @@ -312,12 +322,12 @@ jobs: body: ${{ inputs.release_notes || '' }} generate_release_notes: ${{ inputs.release_notes == '' }} - # ── 7. Verify + Publish (requires security gate) ─────────────── + # ── 7. Verify + Publish (requires the security and CodeQL gates) ─ verify: - needs: [release-draft, security] + needs: [release-draft, security, codeql] # Explicit condition: a deliberately skipped test ancestor must not skip # the publish chain; the draft itself proves smoke and soak both succeeded. - if: ${{ !cancelled() && !failure() && needs.security.result == 'success' && needs.release-draft.result == 'success' }} + if: ${{ !cancelled() && !failure() && needs.security.result == 'success' && needs.codeql.result == 'success' && needs.release-draft.result == 'success' }} runs-on: ubuntu-latest permissions: contents: write diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d749762b6..6c4b07368 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -52,7 +52,7 @@ Operational authority is stricter than code review authority. | Operation | Current authority | Project rule | | --- | --- | --- | -| PR validation (`pr.yml`, DCO, CodeQL) | Automatic | Anyone may trigger it by opening or updating a PR. Required checks must pass. | +| PR validation (`pr.yml`, DCO, PR CodeQL scan) | Automatic | Anyone may trigger it by opening or updating a PR. Required checks must pass. | | Dry run (`dry-run.yml`) | `@DeusData` | Delegated release operators may run dry runs after promotion. Dry-run delegation does not imply release authority. | | Smoke/soak/repro manual runs | `@DeusData` | Area reviewers may operate these runs when delegated for diagnosis. Results are advisory. | | Release workflow (`release.yml`) | `@DeusData` only | Owner-only until a release operator is explicitly promoted. Publishing, replacing releases, and tag movement remain owner-gated. | diff --git a/Makefile.cbm b/Makefile.cbm index 9fc93b73d..443946c7f 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -1299,6 +1299,16 @@ $(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c $(BUILD_DIR)/.build-co OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ) +# The part of OBJS_VENDORED_PROD that holds no first-party code: everything but +# prod_lsp_all.o (first-party LSP unity) and prod_preprocessor.o (first-party +# code around vendored simplecpp). CodeQL's light PR scan builds these BEFORE +# it starts tracing (scripts/build.sh --vendored-only), so the traced build +# (--reuse-vendored) compiles first-party code only: the grammars alone are +# most of the 1.3 GB of vendored C, against ~45 MB of first-party code. +OBJS_THIRD_PARTY_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ) +.PHONY: cbm-third-party +cbm-third-party: $(OBJS_THIRD_PARTY_PROD) + MAIN_SRC = src/main.c # Rebuild when the build CONFIGURATION changes, not only when sources do. The @@ -1376,6 +1386,7 @@ LLVM_BIN := $(shell brew --prefix llvm 2>/dev/null)/bin CLANG_TIDY ?= $(shell [ -x "$(LLVM_BIN)/clang-tidy" ] && echo "$(LLVM_BIN)/clang-tidy" || echo clang-tidy) CLANG_FORMAT ?= $(shell [ -x "$(LLVM_BIN)/clang-format" ] && echo "$(LLVM_BIN)/clang-format" || echo clang-format) CPPCHECK ?= cppcheck +CPPCHECK_JOBS ?= 2 # macOS SDK sysroot (needed for Homebrew LLVM to find system headers) SYSROOT = $(shell xcrun --show-sdk-path 2>/dev/null) @@ -1400,10 +1411,16 @@ lint-tidy: lint-tidy-diff: @scripts/lint-tidy-diff.sh "$(CLANG_TIDY)" $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG) -# cppcheck: complementary analysis (config in .cppcheck) +# cppcheck: complementary analysis (config in .cppcheck). A build directory +# retains whole-program analysis with parallel workers; each run owns its state. lint-cppcheck: @echo "=== cppcheck ===" - @$(CPPCHECK) --enable=warning,style,performance,portability \ + @set -e; \ + cppcheck_dir=$$(mktemp -d "$${TMPDIR:-/tmp}/cbm-cppcheck.XXXXXX"); \ + trap 'status=$$?; rm -rf "$$cppcheck_dir" || :; exit "$$status"' EXIT; \ + trap 'exit 129' HUP; trap 'exit 130' INT; trap 'exit 143' TERM; \ + $(CPPCHECK) -j$(CPPCHECK_JOBS) --cppcheck-build-dir="$$cppcheck_dir" \ + --enable=warning,style,performance,portability \ --std=c11 --language=c \ --suppressions-list=.cppcheck \ --error-exitcode=1 \ diff --git a/README.md b/README.md index 74a2eaf71..f12173304 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# codebase-memory-mcp: local code intelligence for developers and AI agents +# codebase-memory-mcp: local code intelligence for AI agents via MCP [![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/release.yml?branch=main&label=main%20CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/release.yml) -[![C tests](https://img.shields.io/badge/C_tests-9388_cases-blue)](#tests) +[![Native and frontend tests](https://img.shields.io/badge/tests-native_%2B_frontend-blue)](#tests) [![MCP tools](https://img.shields.io/badge/MCP_tools-17-blue)](#mcp-tools) [![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-type_resolution-blue)](#hybrid-lsp) @@ -14,27 +14,28 @@ [![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev) [![VirusTotal](https://img.shields.io/badge/VirusTotal-scanned_every_release-brightgreen?logo=virustotal)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![arXiv](https://img.shields.io/badge/arXiv-2603.27277-b31b1b?logo=arxiv)](https://arxiv.org/abs/2603.27277) +[![Join the community on Discord](https://img.shields.io/badge/Discord-Join_the_community-5865F2?logo=discord&logoColor=white)](https://discord.gg/KguGXxg2X) -**Understand your repository, from source code to the systems around it.** codebase-memory-mcp connects code, documentation, configuration, and infrastructure in a persistent local knowledge graph. Use it to find an unfamiliar feature, follow a call across services, trace a deployment back to its source, or assess which parts of a project a change could affect. +**Understand your repository, from source code to the systems around it.** codebase-memory-mcp is an MCP server that connects code, documentation, configuration, and infrastructure in a persistent local knowledge graph. Use it to find an unfamiliar feature, follow a call across services, trace a deployment back to its source, or assess which parts of a project a change could affect. -Give your coding agent structural context through [Model Context Protocol](https://modelcontextprotocol.io/) (MCP), query the graph from the CLI, or explore it yourself in **CBM Lens Beta**. Lens brings together an IDE-style source reader, architecture maps, the Galaxy graph, coverage diagnostics, architecture decisions, and optional local code chat. +Give your coding agent structural context through [Model Context Protocol](https://modelcontextprotocol.io/) (MCP), or query the graph directly from the CLI. **The MCP server is the core of the project.** Its optional visual companion, **CBM Lens, is in open beta (v0.0.1)**, with a source reader, architecture maps, the Galaxy graph, coverage diagnostics, architecture decisions, and local code chat. -The native C backend exposes **17 MCP tools**, parses [languages and file formats](#language-support) with vendored grammars, and adds [Hybrid LSP](#hybrid-lsp) type resolution for supported languages. The installer supports **45 automatic or conditional client integrations**, including client configuration, skills, agent profiles, and applicable hooks. Indexing and graph queries need no hosted service or API key. +The native C backend exposes **17 MCP tools**, parses [languages and file formats](#language-support) with vendored grammars, and adds [Hybrid LSP](#hybrid-lsp) type resolution for supported languages. The installer supports **45 automatic or conditional client integrations**, including client configuration, skills, agent profiles, and applicable hooks. Indexing and graph queries need no hosted service or API key. Focused graph queries can reduce repeated file reads and token use; see the [measured results](#performance). [Quick start](#quick-start) · [Capabilities](#features) · [Documentation](#documentation-connected-to-code) · [Infrastructure](#infrastructure-and-deployment) · [Smart CI](#smart-ci-and-test-impact) · [Explore CBM Lens](#cbm-lens-beta-code-visualization-and-source-navigation) · [Agent setup](#agent-setup-skills-profiles-and-hooks) · [MCP tools](#mcp-tools) > **Local processing:** Source analysis and graph storage run on your machine. CBM Lens can optionally download a model for browser AI after you enable it; inference stays on the device. The installer writes agent configuration for the clients you select. See [Security](#security) for release verification and [SECURITY.md](SECURITY.md) for reporting issues. -**Architecture overview.** Folders become nested platforms, files and symbols become selectable blocks, and arrows show indexed relationships. The recording zooms into the map of this repository, selects `src/pipeline`, opens the area, selects `artifact.c` and opens its source. Height can show source size; hotspot gravity reflects static fan-in. [Still image](docs/images/lens-architecture.png). +**See the graph with CBM Lens (open beta v0.0.1).** Folders become nested platforms, files and symbols become selectable blocks, and arrows show indexed relationships. The recording opens `src/foundation` in the map of this repository, selects `arena.c` and opens its source. Height can show source size; hotspot gravity reflects static fan-in. [Still image](docs/images/lens-architecture.png).

- CBM Lens Beta architecture walkthrough: zoom into the source folders of this repository, open src/pipeline, select artifact.c and open its source + CBM Lens Beta architecture walkthrough: open src/foundation, select arena.c and read its source beside the map

-**Galaxy.** Explore the code knowledge graph in 3D. The recording searches `cbm_pipeline_run`, follows its outgoing calls one layer further, switches to the hierarchy and shows the path to `cbm_store_check_integrity`. Edge pulses show relationship direction; they do not represent live execution. [Current still: two dependency layers around `cbm_max_file_bytes`](docs/images/lens-galaxy.jpg). +**Galaxy.** Explore the code knowledge graph in 3D. The recording follows the outgoing relationships of `cbm_arena_alloc` one layer further, switches to the hierarchy and shows the two-hop call path to `cbm_alloc`. Edge pulses show relationship direction; they do not represent live execution. [Still: two dependency layers around `cbm_max_file_bytes`](docs/images/lens-galaxy.jpg).

- Code knowledge graph in CBM Lens Beta Galaxy: search a function, expand its outgoing calls, switch to the hierarchy and follow a path + Code knowledge graph in CBM Lens Beta Galaxy: expand cbm_arena_alloc, switch to the hierarchy and follow its call path through arena_grow to cbm_alloc

**Explore.** Browse files in an IDE-style workspace, read syntax-highlighted source in Monaco, and keep the code graph alongside it. File impact shows what depends on the open file, and Code details shows the callers and calls of the symbol under the caret. Select code to [ask the local chat about it](#chat-with-your-codebase-locally). @@ -52,11 +53,11 @@ The native C backend exposes **17 MCP tools**, parses [languages and file format -These captures show CBM Lens Beta on 12–13 October 2026, using public source from this repository. See [capture details](docs/images/README.md) for each capture's build, indexed scope and recording setup. +The high-resolution GIFs show CBM Lens Beta on 20 October 2026; the still images were captured on 12–13 October. All show public source from this repository. See [capture details](docs/images/README.md) for each capture's build, indexed scope and recording setup. ## Why codebase-memory-mcp -- **Find the right code with context**: combine symbol, text, and semantic search with callers, dependencies, source snippets, and file outlines. Reduce repeated file exploration; see [Performance](#performance) for scoped measurements. +- **Find the right code with context**: combine symbol, text, and semantic search with callers, dependencies, source snippets, and file outlines. Targeted queries and compact, paged results can reduce token use and repeated file reads; see [Performance](#performance) for scoped measurements. - **Connect documentation to implementation**: index document sections, PDF text pages, architecture decisions, and supported references into code. Find the explanation alongside the symbol. - **Follow the system beyond source files**: trace supported relationships through Docker, Kubernetes, Helm, Kustomize, Terraform/OpenTofu, Terragrunt, Ansible, scripts, Make, and CI configuration. - **Understand a change before running it**: inspect affected symbols, compare indexed snapshots, and find related tests. [Experimental Smart CI](#smart-ci-and-test-impact) combines static relationships and per-test coverage for supported C/C++ test harnesses, falling back to broader runs when evidence is insufficient. @@ -160,15 +161,15 @@ Thin frontends still write immediate startup and session-specific errors to thei All active CBM processes must run the exact same version, executable build, coordination ABI, and canonical cache root. Equivalent `CBM_CACHE_DIR` aliases resolve to the same root; a genuinely different root is rejected while any CBM process is active. MCP servers, hooks, one-shot CLI commands, temporary index workers, and the daemon share a crash-safe OS admission barrier; starting an ordinary conflicting process fails before doing work and records an explicit conflict in `${CBM_CACHE_DIR}/logs/daemon-conflicts.ndjson`. -The native `install`, `update`, and `uninstall` commands are the deliberate exception to that conflict rule. Download, verification, and private same-filesystem staging happen first so a bad candidate never disrupts active work. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build. +The native `install` and `uninstall` commands are the deliberate exception to that conflict rule. For installation, download, verification, and private same-filesystem staging finish before activation so a bad candidate never disrupts active work. The native `update` command prints the installer command that performs the update. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build. Package-manager setup (npm, PyPI, or Go) verifies and caches the native executable for that package. It does not replace the active native installation and therefore does not stop running CBM sessions. When that cached binary is executed, it still enters the same exact-build admission barrier. The shell and PowerShell installers invoke the verified candidate's native `install` command, so they do receive the full account-wide activation guarantee. -The ordinary `cli` mode runs one command locally, but it is not disconnected from the daemon above: it connects to the shared coordination daemon (starting one if none is already running) for that same admission barrier plus per-project locks on graph mutations. Its connection is a `cli_session`: it never registers with the background watcher and never starts a UI, and it holds the admission lease only for the command's lifetime. If the command's own connection is what started the daemon, that daemon exits again once the command closes and no other session is attached; if a daemon was already running, the command simply joins and leaves it exactly as found. While the command is running, a temporary monitor separately lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details. +The ordinary `cli` mode runs one command locally, but it is not disconnected from the daemon above: it connects to the shared coordination daemon (starting one if none is already running) for that same admission barrier plus per-project locks on graph mutations. Its connection is a `cli_session`: it does not register a project watcher or open a browser, and it holds the admission lease only for the command's lifetime. The shared daemon may serve Lens according to its UI configuration and available assets. If the command's own connection is what started the daemon, that daemon exits again once the command closes and no other session is attached; if a daemon was already running, the command simply joins and leaves it exactly as found. While the command is running, a temporary monitor separately lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details. ### CBM Lens Beta: code visualization and source navigation -CBM Lens Beta is the browser interface included in standard native releases and UI-enabled source builds. It builds on [CodeAtlasWeb](graph-ui/README.md) by [Bernhard Jackiewicz](https://github.com/BernhardJackiewicz) (design and tracking in #1964). Lens is served on loopback. Browser AI downloads its optional model only after you enable it; inference stays on-device. Each panel names where its facts come from, and what the index did not record is shown as a gap. Start the local server: +**CBM Lens is an optional visual companion to the MCP server, currently in open beta (v0.0.1).** Standard native releases include its browser interface, so you can launch Lens directly after installation; UI-enabled source builds include it too. It builds on [CodeAtlasWeb](graph-ui/README.md) by [Bernhard Jackiewicz](https://github.com/BernhardJackiewicz) (design and tracking in #1964). Lens is served on loopback. Browser AI downloads its optional model only after you enable it; inference stays on-device. Each panel names where its facts come from, and what the index did not record is shown as a gap. Start the local server: ```bash codebase-memory-mcp daemon start --open --port=9749 @@ -215,9 +216,9 @@ Open **Chat** with the panel button at the top right to ask about the code you a Questions about callers and callees, such as "Who calls cbm_pipeline_run?", are answered from the loaded graph without the model, and the answer says so. When you select a symbol, the chat lists its indexed facts and the lines of its code, and the model adds one sentence; that sentence is left out when it claims something the source does not show. Text written by the model is labelled as such. -The recording shows a question about callers answered from the indexed graph, next to the automatic explanation of the selected symbol. [Still image](docs/images/lens-chat.png). +The recording asks who calls `arena_grow` and what it calls, with both answers listed directly from the indexed graph beside the selected symbol. [Still image of a model explanation and graph answer](docs/images/lens-chat.png). -

Chat with your codebase locally in CBM Lens Beta: a selected symbol, its listed facts and a caller question answered from the indexed graph

+

Chat with your codebase locally in CBM Lens Beta: callers and callees of arena_grow answered directly from the indexed graph

The chat history is kept for 24 hours in this browser, per project. Each new message uses the source currently open in the reader; earlier source snapshots are not silently accumulated, and messages about another file or selection are not sent along. If a prompt is too long for the model, older messages are left out first; if it still does not fit, CBM Lens asks you to select less code instead of cutting the code. This chat explains the context you give it; it cannot edit files or run tools. Check its answers against the source. The small default model words things loosely, especially in languages other than English. @@ -238,7 +239,7 @@ The Agents workspace and activity overlays are hidden in normal builds. To inclu VITE_EXPERIMENTAL_AGENTS=true scripts/build.sh --with-ui ``` -Activity comes from explicitly configured tool hooks. The bundled adapter supports Claude Code PostToolUse; other clients need their own adapter. Recorded events go to the local daemon, and the reader must enable activity before the browser polls them. This does not automatically observe every agent. See the [frontend setup guide](graph-ui/README.md#die-agentenebene-was-live-arbeitende-agenten-auf-dem-graphen-zeigen). +Activity comes from explicitly configured tool hooks. The bundled adapter supports Claude Code PostToolUse; other clients need their own adapter. Recorded events go to the local daemon, and the reader must enable activity before the browser polls them. This does not automatically observe every agent. See the [frontend setup guide](graph-ui/README.md#experimental-agent-activity). @@ -252,13 +253,13 @@ Enable automatic indexing on MCP session start: codebase-memory-mcp config set auto_index true ``` -`auto_index` is off by default. When enabled, new projects are indexed automatically on first connection. The configurable file limit is `config set auto_index_limit 50000`. Previously indexed projects can be watched independently through `auto_watch` (on by default), subject to the session rules below. +`auto_index` is off by default. When enabled, new projects are indexed automatically on first connection. The configurable file limit is `config set auto_index_limit 50000`. Previously indexed projects can be watched through `auto_watch` (on by default), subject to its global setting and session ownership. -The watcher follows MCP sessions: it watches the project each open MCP session is rooted in (the client's working directory) and stops when the last session for that project closes. A repository indexed with `cli index_repository`, or indexed from a session rooted somewhere else, is not watched; re-run `index_repository` after changing it. `index_status` reports this in its `watch` object: `watched`, and either the poll cadence and last scan time or the `reason` it is not watched. +Each initialized, long-lived MCP session can register its root project when it connects and any target from a successful `index_repository` call, including custom project names and roots in another worktree. While `auto_watch` is enabled, these watches belong to the registering sessions and remain while at least one owning session is connected; closing the last owner removes the watch. An async index registers its target only after success and while its requesting session remains connected. CLI indexing and failed MCP indexes do not register watches. Global `auto_watch` controls registration and removes all watches on the next refresh or registration request when set to `false`. `index_status` reports this in its `watch` object: `watched`, and either the poll cadence and last scan time or the `reason` it is not watched. -Watcher registration is controlled separately by `auto_watch` (default `true`). Set `config set auto_watch false` to keep a session from registering its project with the background watcher; useful when working across many projects and you want each session contained to explicit indexing. +Watcher registration is controlled globally by `auto_watch` (default `true`). On the next refresh or registration request, `true` allows MCP sessions to register their root on connect and their successful `index_repository` targets; `false` removes all watches. CLI indexing and failed MCP indexes do not add watches. -To turn the watcher off entirely, set `config set watcher_enabled false` (default `true`): the background poll thread never starts and no project is registered, while `auto_index` and manual `index_repository` keep working. Unlike `auto_watch`, which is consulted per session, `watcher_enabled` is read once when the background daemon starts, so run `codebase-memory-mcp daemon stop` after changing it; reconnecting your MCP client alone will not restart the daemon. See [docs/CONFIGURATION.md](docs/CONFIGURATION.md#2-cli-managed-runtime-settings). +To turn the watcher off entirely, set `config set watcher_enabled false` (default `true`): the background poll thread never starts and no project is registered, while `auto_index` and manual `index_repository` keep working. Unlike `auto_watch`, which takes effect on the next refresh or registration request, `watcher_enabled` is read once when the background daemon starts, so run `codebase-memory-mcp daemon stop` after changing it; reconnecting your MCP client alone will not restart the daemon. See [docs/CONFIGURATION.md](docs/CONFIGURATION.md#2-cli-managed-runtime-settings). The watcher follows **git** projects only by default: a project indexed from a directory that is not a git repository is not watched and stays at its last index until you run `index_repository` again. To have such roots polled too, set `config set watch_non_git true` (default `false`, read when the daemon starts): they are then checked on the same cadence with a file-tree scan that uses the indexer's own ignore rules. @@ -273,7 +274,7 @@ bash "/install.sh" ```powershell # Windows -powershell -ExecutionPolicy Bypass -File "\install.ps1" +powershell -File "\install.ps1" ``` The install script is placed next to the binary at install time, so the printed path resolves beside the executable. It is idempotent, so re-running it *is* the update: it stops the daemon, retires the running binary, installs the new one, and cleans up. @@ -401,7 +402,7 @@ In this repository, test narrowing is wired to the Linux x86_64 test leg when `T ### Distribution & operation - **Native runtime set, zero infrastructure services**: SQLite-backed, persists to `~/.cache/codebase-memory-mcp/` -- **Auto-sync**: Background watcher detects file changes in the project an MCP session is working in and re-indexes it automatically (other indexed repos: re-run `index_repository`) +- **Auto-sync**: Background watcher detects file changes in projects registered by active MCP sessions, including successful explicit `index_repository` targets, and re-indexes them automatically. - **Route nodes**: REST endpoints are first-class graph entities - **CLI mode**: `codebase-memory-mcp cli search_graph --project my-project --name-pattern ".*Handler.*"` - **Available on**: npm, PyPI, Homebrew, Scoop, AUR, `go install` @@ -467,15 +468,15 @@ Published backend benchmarks on Apple M3 Pro. These are separate from the CBM Le | Dead code detection | ~150ms | Full graph scan with degree filtering | | Trace call path (depth=5) | <10ms | BFS traversal | -**RAM-first pipeline**: All indexing runs in memory (LZ4 HC compressed read, in-memory SQLite, single dump at end). Memory is released back to the OS after indexing completes. +**Memory-aware pipeline**: Indexing keeps hot data in memory and can spill extraction results to disk when needed. Memory is released back to the OS after indexing completes. -**Token efficiency, within the reported workloads**: The project's [original README report](https://github.com/DeusData/codebase-memory-mcp/blob/4ff6feeab639c4668da0f491930326ed5ec63093/README.md#performance) lists ~3,400 tokens for five structural queries via codebase-memory-mcp versus ~412,000 via file-by-file exploration: approximately **120× fewer tokens** for that comparison. The separate [31-repository preprint](https://arxiv.org/html/2603.27277v1#S4.SS1) reports **10× lower token use** and **2.1× fewer tool calls**, with answer-quality scores of **83% for CBM versus 92% for file-by-file exploration**. These are scoped backend measurements, not a promised saving for every task or a benchmark of CBM Lens. Actual savings depend on the question, repository, agent, and required verification. The study's [system-performance measurements](https://arxiv.org/html/2603.27277v1#S4.SS3) include the Linux indexing and Cypher timings above. +**More focused context, fewer tokens**: The project's [31-repository preprint](https://arxiv.org/html/2603.27277v1#S4.SS1) reports **10× lower token use** and **2.1× fewer tool calls** compared with file-by-file exploration. These March 2026 backend measurements cover the study's workloads and agent setup; the paper documents the full evaluation. An earlier [five-query comparison](https://github.com/DeusData/codebase-memory-mcp/blob/4ff6feeab639c4668da0f491930326ed5ec63093/README.md#performance) reported ~3,400 versus ~412,000 tokens for structural queries. The study's [system-performance measurements](https://arxiv.org/html/2603.27277v1#S4.SS3) include the Linux indexing and Cypher timings above. -To measure comparable quality, latency, and agent-efficiency metrics on your own workload, see [Measuring quality, latency, and agent savings](docs/MEASURING_SAVINGS.md). Exact reproduction of the figures above requires the original inputs and raw artifacts. +Explore the [measurement guide](docs/MEASURING_SAVINGS.md) to evaluate quality, latency and token use on your own workload. ## Troubleshooting & Diagnostics -codebase-memory-mcp runs **100% locally and collects no telemetry**: your code, queries, environment, and usage never leave your machine. That privacy guarantee also means that when you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no data at all unless you choose to send it.** Here is how to capture it yourself. +codebase-memory-mcp **indexes and queries locally and collects no telemetry**. When you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no diagnostic data unless you choose to send it.** Here is how to capture it yourself. ### Capture a diagnostics log @@ -614,7 +615,7 @@ Counts on `main` at `4d9592b5`, where CBM Lens arrived. Native and Go counts use | npm package wrapper | 24 tests | Counted in the source | | Python package wrapper | 43 tests | Counted in the source | -Run the test suite (9,388 tests across 189 suites): +Run the native test suite: ```bash scripts/test.sh # full: clean sanitizer build + all suites + guards @@ -864,7 +865,7 @@ codebase-memory-mcp cli search_graph --project my-project --label Function --for codebase-memory-mcp cli list_projects --format json --detail stats | jq '.projects[].name' ``` -JSON arguments can also be piped on stdin, for tools that take arguments. A tool whose input schema declares none; `list_projects`; never reads stdin, so it stays responsive when it inherits a pipe the caller never closes (the default for `child_process.spawn` and similar wrappers). Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin. +Tools requiring arguments accept JSON piped on stdin. Tools with only optional arguments, such as `list_projects`, accept flags, `--args-file`, or redirected JSON files without waiting on an inherited pipe. Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin. ## MCP Tools @@ -872,8 +873,8 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | Tool | Description | |------|-------------| -| `index_repository` | Index a repository into the graph. Auto-sync keeps it fresh only while it is the project of an open MCP session; repos indexed from the CLI or from another project's session need a re-run (`index_status` shows `watch`). Waits for the whole index by default; pass `async: true` to start it in the daemon and return at once, then poll with `status: true` (see below). | -| `list_projects` | List all indexed projects with node/edge counts. | +| `index_repository` | Index a repository into the graph. A successful call from an active MCP session registers the target for auto-sync while that session remains connected, subject to the global `auto_watch` setting; CLI calls do not register watches. Waits for the whole index by default; pass `async: true` to start it in the daemon and return at once, then poll with `status: true` (see below). | +| `list_projects` | List indexed projects with paging; use `detail: "stats"` for node/edge counts and database sizes. | | `delete_project` | Remove a project and all its graph data. | | `index_status` | Check indexing status of a project. | | `check_index_coverage` | Check whether exact paths or a scope are indexed and fresh. A clean result means no recorded gap, not proof of completeness. | @@ -966,7 +967,7 @@ See [docs/cbmignore.md](docs/cbmignore.md) for the full `.cbmignore` how-to: syn codebase-memory-mcp config list # show all settings codebase-memory-mcp config set auto_index true # auto-index on session start codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index -codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) +codebase-memory-mcp config set auto_watch false # remove/disable all background watches (default: true) codebase-memory-mcp config set doc_candidates false # skip doc<->code candidate scoring on huge repos (default: auto = on for full, off for moderate/fast) codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true) codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) @@ -989,7 +990,7 @@ is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). | `CBM_DOWNLOAD_URL` | *(GitHub releases)* | Override the download URL for updates. Used for testing or self-hosted deployments. | | `CBM_LOG_LEVEL` | role-aware | Set the minimum log level. Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and its supervised index workers default to `info` so lifecycle and liveness records remain available. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none`; or their numeric equivalents `0` to `4`. A physical worker retains INFO liveness records even under a stricter override because its private log drives the supervisor's no-progress timeout. Frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. | | `CBM_WORKERS` | *(detected)* | Override the parallel-indexing worker count returned by `cbm_default_worker_count`. Useful inside containers where `sysconf(_SC_NPROCESSORS_ONLN)` reports host CPUs rather than the cgroup's effective quota. Range 1 to 256; invalid values are ignored with a warning. | -| `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the `ram_fraction × total_RAM` default. Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). | +| `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the derived default (`ram_fraction × total_RAM`, adjusted for available memory and daemon worker caps). Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). | | `CBM_DUMP_VERIFY_MIN_RATIO` | `0.5` | After indexing, compare persisted SQLite node count to the in-memory dump count. When persisted nodes fall below this fraction of committed nodes (and committed > 50), `index_repository` returns `status:"degraded"` instead of silent `indexed`. Range 0 to 1; set `0` to disable. Invalid values are ignored with a warning. | Environment used by daemon-owned components (such as diagnostics, daemon logging, and process-wide indexing resource limits) is captured from the first daemon-backed session that starts the daemon. Later sessions join that process and cannot replace those values. To change them, close all daemon-backed sessions, update the relevant agent configurations consistently, and restart a session. `CBM_ALLOWED_ROOT` remains session-specific, a conflicting `CBM_CACHE_DIR` is rejected, and a one-shot CLI command is not exempt from the rule above: it connects to the coordination daemon like any other session, starting one if none is running. Its own environment becomes the captured daemon-owned environment only when its invocation is the one that starts the daemon; joining an already-running daemon, it inherits that daemon's already-captured values instead. @@ -1019,9 +1020,9 @@ Need the full config-file reference? See [docs/CONFIGURATION.md](docs/CONFIGURAT Each entry maps an extension (which **must** start with `.`) to a language name. Language names are matched **case-insensitively**. Accepted values (aliases in parentheses) are: -`bash` (`sh`), `c`, `c++` (`cpp`), `c#` (`csharp`), `clojure`, `cmake`, `cobol`, `common lisp` (`commonlisp`, `lisp`), `css`, `cuda`, `dart`, `dockerfile`, `elixir`, `elm`, `emacs lisp` (`emacslisp`), `erlang`, `f#` (`fsharp`), `form`, `fortran`, `glsl`, `go`, `graphql`, `groovy`, `haskell`, `hcl` (`terraform`), `html`, `ini`, `java`, `javascript`, `json`, `julia`, `kotlin`, `lean`, `lua`, `magma`, `makefile`, `markdown`, `matlab`, `meson`, `nix`, `objective-c` (`objc`), `ocaml`, `perl`, `php`, `protobuf`, `python`, `r`, `ruby`, `rust`, `scala`, `scss`, `sql`, `svelte`, `swift`, `toml`, `tsx`, `typescript`, `verilog`, `vimscript`, `vue`, `wolfram`, `xml`, `yaml`, `zig`. +`arkts`, `asciidoc`, `bash` (`sh`), `c`, `c++` (`cpp`), `c#` (`csharp`), `chialisp`, `clojure`, `cmake`, `cobol`, `common lisp` (`commonlisp`, `lisp`), `css`, `cuda`, `dart`, `diagram`, `dockerfile`, `elixir`, `elm`, `emacs lisp` (`emacslisp`), `erlang`, `f#` (`fsharp`), `form`, `fortran`, `glsl`, `go`, `graphql`, `groovy`, `haskell`, `hcl` (`terraform`), `html`, `ini`, `java`, `javascript`, `jinja2` (`jinja`), `json`, `julia`, `kotlin`, `lean`, `lua`, `magma`, `makefile`, `markdown`, `matlab`, `meson`, `mojo`, `nix`, `objective-c` (`objc`), `ocaml`, `openapi`, `pdf`, `perl`, `php`, `plsql`, `protobuf`, `python`, `r`, `ruby`, `rust`, `scala`, `scss`, `sql`, `svelte`, `swift`, `toml`, `tsx`, `typescript`, `verilog`, `vimscript`, `vue`, `wolfram`, `xml`, `yaml`, `zig`. -Project config overrides global for conflicting extensions. An entry whose language name is unknown, or whose extension does not start with `.`, is skipped and a warning is logged to stderr (shown at the default `info` log level). Missing config files are ignored. +Project config overrides global for conflicting extensions. An entry whose language name is unknown, or whose extension does not start with `.`, is skipped and logged as a warning. Missing config files are ignored. ## Persistence @@ -1111,7 +1112,7 @@ Every release is verified through a multi-layer pipeline before publication: - **Sigstore cosign**: keyless signatures on all artifacts; bundles included in every release - **SHA-256 checksums**: `checksums.txt` published with every release; verified by both install scripts before extraction - **CodeQL SAST**: blocks release pipeline if any open alerts remain -- **No language-runtime dependency chain**: the native backend uses libraries vendored at compile time, and the frontend is embedded by the UI build +- **No language-runtime dependency chain**: the native backend needs no language-runtime environment; UI builds embed the frontend ### VirusTotal release scans @@ -1130,6 +1131,10 @@ The [v0.11.0 release notes](https://github.com/DeusData/codebase-memory-mcp/rele The release also publishes [candidate scan results](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/virustotal-candidate-results.tsv), [binary provenance](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/release-candidates.tsv), and [selection records](https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.11.0/release-selection.tsv). For subsequent versions, use the scan evidence linked from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest). +## Help shape what comes next + +The next idea could be yours. We invite the OSS community to bring ideas, tricky integration questions, and feedback to [our Discord community](https://discord.gg/KguGXxg2X). When a direct conversation works better, reach out to **DeusData** or the project's co-developers there. Let's build what comes next together. + ## License MIT diff --git a/SECURITY.md b/SECURITY.md index 48da1d1a1..ec9fea712 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -111,7 +111,7 @@ This project implements multiple layers of security verification. Every release - **All dangerous function calls** require a reviewed entry in `scripts/security-allowlist.txt` - **Time-bomb pattern detection** — scans for `time()`/`sleep()` near dangerous calls (could indicate delayed activation) - **MCP tool handler file read audit** — tracks file read count in `mcp.c` against an expected maximum (detects added file reads that could exfiltrate data through tool responses) -- **CodeQL SAST** — static application security testing on every push (taint analysis, CWE detection, data flow tracking). Any open alert blocks the release. +- **CodeQL SAST** — static application security testing of the C/C++, JavaScript/TypeScript, Python and GitHub Actions code (taint analysis, CWE detection, data flow tracking): a full scan on every push to main, weekly, and in every dry run and release; a first-party scan of the languages a pull request changes, with alerts reported on its changed lines. Any open alert fails the run and blocks the release. - **Fuzz testing** — random/mutated inputs to MCP server and Cypher parser (60 seconds per build). Catches crashes, segfaults, and memory errors that structured tests miss. - **Native antivirus scanning** on every platform (any detection fails the build): - **Windows**: Windows Defender with ML heuristics — the same engine end users run diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 04afa5b3e..f2ebf78eb 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -87,9 +87,9 @@ Current keys: |---|---|---| | `auto_index` | `false` | Automatically index new projects when an MCP session starts. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | -| `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). | +| `auto_watch` | `true` | Global switch for MCP session watch registration. On the next refresh or registration request, `true` allows session roots and successful `index_repository` targets to be watched; `false` removes all watches. The background watcher can still run with no registered projects. | | `doc_candidates` | `auto` | Score documentation sections against code to store doc↔code candidates (the possible targets of a section, and the sections that may describe a function or file, each with a judged probability). Unset (`auto`), the index mode decides: on for `full` indexes (the default mode), off for `moderate` and `fast`. Set `true` or `false` to choose for every mode; `config reset doc_candidates` returns to `auto`. The scoring grows with sections × functions: under a second up to ~100k functions, ~15 s at ~400k, ~3 minutes on the Linux kernel. Off stores no candidates and leaves the rest of the graph unchanged; on in `fast` mode stores candidates without adding `fast`'s skipped `SIMILAR_TO` / `SEMANTICALLY_RELATED` edges. `index_repository` also takes `doc_candidates` for a single index, which wins over this setting. Applies from the next index. | -| `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. Only an open MCP session's own project is ever watched; `index_status` reports the current state in its `watch` object. | +| `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. Active MCP sessions can register their root project and successful explicit index targets; `index_status` reports the current state in its `watch` object. | | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | @@ -102,15 +102,23 @@ has not run yet). > **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the > watcher *subsystem* starts at all (the background poll thread). `auto_watch` is -> narrower: it only controls whether a connecting session registers *its own* -> project with an already-running watcher. When `watcher_enabled=false`, +> narrower: it globally controls MCP session watch registration. On the next +> refresh or registration request, `false` removes all watches; `true` allows +> session roots and successful explicit `index_repository` targets to register. +> Watcher refreshes retain `mode`, `persistence`, and the resolved +> `doc_candidates` boolean from the latest successful explicit index registered +> for that target. This boolean can come from the request or its global setting. +> When neither supplied a boolean, refresh uses the current setting or mode default. +> A watcher callback also observes `auto_watch=false`: it removes all watches +> before admitting any new refresh work. +> When `watcher_enabled=false`, > `auto_watch` has no effect — there is no watcher to register with. > > They also differ in **when they are read**, which matters because the watcher > lives in the background daemon, not in your MCP client: > -> - `auto_watch` is consulted each time a session would register its project, so -> a change applies to sessions that connect afterwards. +> - `auto_watch` takes effect on the next refresh or registration request and +> applies globally to session roots and successful `index_repository` targets. > - `watcher_enabled` is read **once, when the daemon starts**, because it decides > whether the watcher is built at all. The daemon is long-lived and outlives > individual MCP sessions, so **reconnecting your client is not enough** — retire diff --git a/docs/images/README.md b/docs/images/README.md index 1c1ecc96d..89f811c37 100644 --- a/docs/images/README.md +++ b/docs/images/README.md @@ -1,49 +1,53 @@ # CBM Lens Beta README captures -These are screenshots and browser recordings of the application, captured on 12–13 October 2026. The application data and text were not retouched. Bernhard Jackiewicz captured the recordings and PNG stills on 12 October. The Explorer and Galaxy JPEG stills were refreshed on 13 October. One addition is drawn on top of the recordings: a small mouse pointer and click ring, because a headless browser shows no pointer of its own. +These are screenshots and browser recordings of the real application using public source from this repository. The application data and text were not retouched. The GIFs were refreshed on 20 October 2026; the still images preserve the earlier captures from 12–13 October. -## Original captures: 12 October +## High-resolution recordings: 20 October - Repository: [DeusData/codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp) -- Branch: `main`, after [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068) brought CBM Lens Beta in -- Commit: `4d9592b5` -- Build: `scripts/build.sh --with-ui` -- Browser: headless Chromium (Playwright) on macOS, English locale, viewport 1440 × 900, device scale factor 1 -- Agents: standard build, experimental agent activity disabled +- Source commit: `4cf44147e639ae070ee44d9230244bfb9d9676e0` +- Build: `make -f Makefile.cbm cbm-with-ui`, with the standard frontend and experimental agent activity disabled +- Capture: built-in browser screenshots on macOS, English locale, native viewport **1600 × 1000**, **12 frames per second** +- Encoding: JPEG screenshot frames converted to GIF with one 256-color palette per recording, no dithering, infinite looping; no upscaling, sharpening or pointer overlays -The `v0.0.1` label in the application is the frontend package version. It is not the backend release version. +The `v0.0.1` label is the frontend package version, not the backend release version. Standard native releases include Lens; using a source build for these recordings is a capture-provenance detail, not an installation requirement. -### Data shown +### Data and interaction -The project `codebase-memory-mcp` is an export of the same public commit (`git archive`), indexed in `full` mode into an isolated local cache. No private repository was used. Because the export has no `.git` folder, Explore reports "Git unavailable" in its file impact line. +The project `codebase-memory-mcp` is a clean `git archive` export of the source commit, indexed in `full` mode into an isolated local cache. No private repository was used. Because the export has no `.git` folder, the interface reports "Git unavailable" where applicable. -The index reported 75,037 nodes and 361,349 edges. These are properties of this source scope, not a benchmark. The interface keeps its coverage and display-limit notices. +The index reported 81,190 nodes and 411,072 edges. These counts describe this source scope, not a benchmark or proof of complete repository coverage. The interface retains its coverage and display-limit notices. The clips join recorded interaction segments with pauses shortened between them; their displayed application states and answers are unchanged. Directional pulses represent indexed relationships, not observed execution. -## Refreshed stills: 13 October +| Recording | Duration | What it shows | +|-----------|----------|---------------| +| [lens-overview.gif](lens-overview.gif) | 12.5 seconds | Architecture overview, opening `src/foundation`, browsing its files, selecting `arena.c` and reading its source beside the map. | +| [lens-galaxy.gif](lens-galaxy.gif) | 12 seconds | Expanding outgoing relationships around `cbm_arena_alloc`, switching to the hierarchy and tracing the two-hop call path `cbm_arena_alloc → arena_grow → cbm_alloc`. | +| [lens-chat.gif](lens-chat.gif) | 11.5 seconds | `arena_grow` selected in Galaxy, with caller and callee questions answered directly from the indexed graph. | + +All three GIFs are 1600 × 1000. Their frames and playback duration were decoded and checked after export. README and website use these same files. + +### Local chat check + +The optional Qwen2.5 Coder 0.5B model was downloaded and loaded through the application's own setup. Automatic explanations were disabled for this recording. Both displayed answers are deterministic graph answers, labelled "Listed from the indexed graph; not generated by the model." The clip shows one incoming `CALLS` relationship from `cbm_arena_alloc` and two outgoing `CALLS` relationships to `cbm_alloc` and `cbm_free`, alongside the other indexed relationship types. It does not demonstrate model-generated prose. + +## Original still images: 12 October + +Bernhard Jackiewicz captured the PNG stills on 12 October 2026, from commit `4d9592b5` on `main` after [PR #2068](https://github.com/DeusData/codebase-memory-mcp/pull/2068). The build used `scripts/build.sh --with-ui`; headless Chromium on macOS used English locale, a 1440 × 900 viewport and device scale factor 1. Experimental agent activity was disabled. + +The indexed project was a public `git archive` export of that commit in `full` mode, with 75,037 nodes and 361,349 edges. The PNG stills are 1440 × 900, except the chat still at 1440 × 1200 so the explanation and answer fit together. These are historical application views, also used as reduced-motion alternatives on the website; they are not extracted from the new GIFs. + +| Still | What it shows | +|-------|---------------| +| [lens-architecture.png](lens-architecture.png) | The whole repository on the architecture map. | +| [lens-chat.png](lens-chat.png) | A local model explanation of `cbm_pipeline_pass_doclinks` and a caller answer listed from the graph. | +| [lens-system-structure.png](lens-system-structure.png) | Component groups and their dependencies in System structure, with `src` selected. | +| [lens-behavior.png](lens-behavior.png) | Possible direct calls of `compileCards`, with its declaration. | + +## Refreshed still images: 13 October `lens-explore.jpg` and `lens-galaxy.jpg` are unedited browser captures at 1280 × 720 from the locally served CBM Lens Beta UI. The running binary reports `dev`; its exact build commit is unavailable, so these captures do not certify an exact source revision. The local project is named `cbm-triage-main-20260919`; the screenshots show public CBM source and indexed relationships. -- **Explore:** `src/foundation/limits.c`, its compact file impact summary and direct dependencies (24 nodes, 26 edges). The UI's freshness, sampling and coverage caveats remain visible. -- **Galaxy:** `cbm_max_file_bytes`, both directions, all edge types, two layers (160 nodes, 218 edges). +- **[Explore](lens-explore.jpg):** `src/foundation/limits.c`, its compact file impact summary and direct dependencies (24 nodes, 26 edges). The UI's freshness, sampling and coverage notices remain visible. +- **[Galaxy](lens-galaxy.jpg):** `cbm_max_file_bytes`, both directions, all edge types, two layers (160 nodes, 218 edges). -These counts describe the displayed index snapshot, not a benchmark or proof of complete repository coverage. The original recordings and other stills are unchanged. - -## Media - -| File | What it shows | -|------|---------------| -| [lens-overview.gif](lens-overview.gif) | Architecture overview of the repository: zoom and a small orbit, selecting `src/pipeline`, opening that area, selecting `artifact.c` and opening its source. | -| [lens-architecture.png](lens-architecture.png) | Still image of the whole repository on the architecture map. | -| [lens-galaxy.gif](lens-galaxy.gif) | Galaxy: searching `cbm_pipeline_run`, following its outgoing calls one layer further, the hierarchy view and the path to `cbm_store_check_integrity`. | -| [lens-galaxy.jpg](lens-galaxy.jpg) | Two dependency layers around `cbm_max_file_bytes`, with direction and edge-type controls. | -| [lens-explore.jpg](lens-explore.jpg) | `src/foundation/limits.c` in the reader with its file impact summary, file tree and direct graph dependencies. | -| [lens-chat.gif](lens-chat.gif) | The chat with the local Qwen2.5 Coder 0.5B model: the explanation of `cbm_pipeline_pass_doclinks` and "Who calls cbm_pipeline_pass_doclinks?" answered from the graph. | -| [lens-chat.png](lens-chat.png) | Still image of the same explanation and listed answer. | -| [lens-system-structure.png](lens-system-structure.png) | Component groups and their dependencies in System structure, `src` selected. | -| [lens-behavior.png](lens-behavior.png) | Possible direct calls of `compileCards`, with its declaration. | - -The recordings were captured as browser video, then encoded as GIF at 960 pixels wide and 10 frames per second with a palette per recording. While a search is typed, the Galaxy briefly shows "No matching indexed nodes." until the index search answers; those frames (under one second) were cut. The original PNG stills are 1440 × 900, the chat still 1440 × 1200 so that the explanation and the answer fit together. The refreshed JPEG stills are 1280 × 720. Directional pulses represent indexed relationships, not observed execution. - -## Local chat check - -The chat recording uses the actual browser model, downloaded through the application's own setup. The explanation lists the indexed facts and the declaration of the selected function; the one sentence after them was written by the model and is labelled as such. The caller answer is listed from the indexed graph without the model. +These counts describe the displayed index snapshot, not a benchmark or proof of complete repository coverage. diff --git a/docs/images/lens-chat.gif b/docs/images/lens-chat.gif index 1eaaecb86..946f777e7 100644 Binary files a/docs/images/lens-chat.gif and b/docs/images/lens-chat.gif differ diff --git a/docs/images/lens-galaxy.gif b/docs/images/lens-galaxy.gif index 1aa8899e3..3c906eea1 100644 Binary files a/docs/images/lens-galaxy.gif and b/docs/images/lens-galaxy.gif differ diff --git a/docs/images/lens-overview.gif b/docs/images/lens-overview.gif index 733833553..058397d1e 100644 Binary files a/docs/images/lens-overview.gif and b/docs/images/lens-overview.gif differ diff --git a/docs/index.html b/docs/index.html index 2c48b3910..fac7e3b5b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,55 +3,103 @@ - Codebase Memory — Connect your code, docs and infrastructure - + Codebase Memory — Local Code Intelligence & MCP + - - + + + + + - - + + + +