#!/usr/bin/env bash # The code-scanning gate must FAIL CLOSED -- and CodeQL must run where it gates. # # History. `codeql-gate` in .github/workflows/_security.yml (since moved into # .github/workflows/codeql.yml as the analyze job's alert check) counted open # alerts with: # # ALERTS=$(gh api '.../code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") # # That reads as defensive and is the exact opposite. The job declared no # `permissions:` block, so it inherited the workflow's `contents: read` and the # alert API answered 403. `gh` writes the API error BODY to stdout, so `ALERTS` # became the string # # {"message":"Resource not accessible by integration",...,"status":"403"}0 # # every `[ ... -gt ... ]` then failed as a non-integer comparison, the `if` took # its false branch, and the step printed "CodeQL gate passed (0 alerts)" and # exited 0. Observed live in a GREEN run (34700514102 / job 103571322660), while # the repository had an open high-severity alert. # # A security gate that cannot read its input must stop the build, not wave it # through. This test pins both halves of that: the permission that makes the # read possible, and the fail-closed handling for when it still is not. # # It also pins where CodeQL runs. The former PR gate polled up to 150 min per # PR for a separate CodeQL run and starved PR CI of runners; now: # - no pull_request trigger and no job that polls for a CodeQL run; # - PR CI calls codeql.yml as its own job (needed by ci-ok), one leg per # selected codeql-* lane; each leg analyses its language's whole # first-party scope (never narrowed by the diff); c-cpp compiles the # vendored objects before tracing, then traces the first-party build plus # the vendored files the PR changes; # - push to main, the dry run and the release run the full scan, and the # release's verify needs it; # - superseded push and PR runs are cancelled, dispatched ones never. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" python3 - "$ROOT" <<'PY' from __future__ import annotations import json import pathlib import re import sys root = pathlib.Path(sys.argv[1]) failures: list[str] = [] def require(condition: bool, message: str) -> None: if not condition: failures.append(message) workflow_dir = root / ".github" / "workflows" def text(name: str) -> str: return (workflow_dir / name).read_text(encoding="utf-8") def job_body(source: str, job: str) -> str | None: m = re.search(rf"\n {re.escape(job)}:\n(?P(?: .*\n|\n)*)", source) return m.group("body") if m else None def code_only(source: str) -> str: return "\n".join(line for line in source.splitlines() if not line.lstrip().startswith("#")) def steps(body: str) -> list[tuple[str, str]]: """(name, step text) per `- name:`/`- uses:` step of a job body.""" found = [] for m in re.finditer(r"(?ms)^ - (?P.*?)(?=^ - |\Z)", body): step = m.group("step") name = re.search(r"(?m)^(?:name: |\s+name: )(.*)$", step) found.append((name.group(1).strip() if name else step.splitlines()[0], step)) return found workflow = text("codeql.yml") gate = job_body(workflow, "analyze") require(gate is not None, "codeql.yml no longer defines an analyze job") body = gate or "" # 1. The read has to be possible at all. require( re.search(r"^ permissions:\s*$", body, re.M) is not None, "analyze must declare its own permissions block -- inheriting the " "workflow default leaves it without security-events access and the alert " "API answers 403", ) require( re.search(r"^ security-events:\s*(read|write)\s*$", body, re.M) is not None, "analyze must hold security-events (write implies read), or the " "code-scanning alert query 403s and the gate cannot see any alert", ) # 2. The failure direction. `|| echo "0"` on the alert query is the specific # construct that turned an API error into a clean pass. require( re.search(r"code-scanning/alerts[^\n]*\|\|\s*echo", body) is None, "the alert count must not fall back to a literal on error -- an unreadable " "alert list is not zero alerts", ) require( "BLOCKED: cannot read code scanning alerts" in body, "the alert check must fail with a named error when the alert API cannot be read", ) require( re.search(r"alert count was not a number", body) is not None, "the alert check must reject a non-numeric alert count instead of comparing it", ) # 3. The count is produced in a command substitution, so a bare `exit` inside # the helper would end only the subshell and let the caller proceed with an # empty value -- the same class of silent pass. The callers must check. assignments = re.findall(r"^\s*ALERTS2?=\$\(count_open_alerts\)(.*)$", body, re.M) require( len(assignments) >= 2, "expected both alert samples to go through the checked helper", ) require( all("|| exit 1" in tail for tail in assignments), "every count_open_alerts call must be followed by || exit 1 -- a failure " "inside a command substitution does not stop the caller on its own", ) # 4. Triggers: never per pull request (PR CI calls the workflow instead), on # push to main, callable by PR CI, the dry run and the release. on = re.search(r"(?ms)^on:\n(?P.*?)(?=^\S)", workflow) triggers = set(re.findall(r"(?m)^ ([a-z_]+):", on.group("body"))) if on else set() require("pull_request" not in triggers and "pull_request_target" not in triggers, "codeql.yml must not trigger on pull requests (pr.yml calls it as a job)") require({"push", "workflow_call"} <= triggers, f"codeql.yml must run on push to main and be callable, has {sorted(triggers)}") require(re.search(r"(?m)^ push:\n branches: \[main\]$", workflow) is not None, "codeql.yml's push trigger must be main") # 5. Nothing waits for a separate CodeQL run, and _security.yml has no CodeQL. for path in sorted(workflow_dir.glob("*.yml")): require("workflows/codeql.yml/runs" not in code_only(path.read_text(encoding="utf-8")), f"{path.name} polls for a CodeQL run -- CodeQL runs as a job of the caller") require("codeql" not in code_only(text("_security.yml")).lower(), "_security.yml must not run or wait for CodeQL") # 6. Superseded runs are cancelled (push to main, a PR), dispatched never; # legs of one run never cancel each other. concurrency = re.search(r"(?ms)^ concurrency:\n(.*?)(?=^ \S)", body) conc = concurrency.group(1) if concurrency else "" require("matrix.language" in conc, "the concurrency group must be per language") require("github.event_name == 'push' && github.ref" in conc, "push runs must share a ref-scoped group (only the newest main commit is scanned)") require("github.event.pull_request.number" in conc, "PR scans must share a per-PR group") require("cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}" in conc, "only push and PR scans may be cancelled -- a dry run or release tests one commit") # 7. The PR scan: analysis never narrowed by the diff; c-cpp prebuilds the # vendored objects untraced and traces first-party + changed vendored. names = [name for name, _ in steps(body)] by_name = dict(steps(body)) def index(fragment: str) -> int: hits = [i for i, n in enumerate(names) if fragment in n] return hits[0] if hits else -1 init, analyze = index("Initialize CodeQL"), index("Perform CodeQL Analysis") prebuild, retrace = index("Build third-party objects"), index("Vendored files this PR changes") light_build, full_build = index("PR scan, first-party"), index("Build for CodeQL analysis (full)") require(min(init, analyze, prebuild, retrace, light_build, full_build) >= 0, f"codeql.yml analyze lost a step, has {names}") require(-1 < prebuild < init and -1 < retrace < init, "the vendored prebuild and the retrace list must run BEFORE Initialize CodeQL (untraced)") require(init < light_build < analyze and init < full_build < analyze, "the traced builds must run between Initialize CodeQL and the analysis") for name in names: if "Initialize CodeQL" in name or "Perform CodeQL Analysis" in name or "open code scanning alerts" in name: require(not re.search(r"(?m)^\s+if:", by_name[name]), f"step '{name}' must never be skipped -- a selected leg always analyses") light_step = by_name.get(names[light_build], "") if light_build >= 0 else "" require("scripts/build.sh --reuse-vendored --retrace-from" in light_step, "the PR c-cpp build must reuse the untraced vendored objects and retrace the changed ones") require("scripts/ci/codeql-retrace-vendored.sh" in (by_name.get(names[retrace], "") if retrace >= 0 else ""), "the retrace list must come from scripts/ci/codeql-retrace-vendored.sh") require("scripts/build.sh --vendored-only" in (by_name.get(names[prebuild], "") if prebuild >= 0 else ""), "the PR c-cpp prebuild must be scripts/build.sh --vendored-only") require(re.search(r"(?m)^\s+run: scripts/build\.sh\s*$", by_name.get(names[full_build], "") if full_build >= 0 else "") is not None, "the full c-cpp scan must trace the whole production build (vendored C included)") require(re.search(r'(?m)^ CBM_NO_CCACHE: "1"$', body) is not None, "the CodeQL job must disable the compiler cache -- a cache hit compiles nothing to trace") require("config-file: ./.github/codeql/codeql-config.yml" in body, "every scan must use the first-party scope in .github/codeql/codeql-config.yml") # 8. One configuration per language: the full matrix and PR CI's legs agree. default = re.search(r"fromJSON\(inputs\.matrix \|\| '(?P\{.*?\})'\)", body) pr = text("pr.yml") pr_legs = re.search(r"CODEQL='(?P\[.*?\])'", pr) full_legs = json.loads(default.group("json"))["include"] if default else [] require({leg.get("language") for leg in full_legs} == {"c-cpp", "javascript-typescript", "python", "actions"}, "the full scan must cover c-cpp, javascript-typescript, python and actions") require(pr_legs is not None and json.loads(pr_legs.group("json")) == full_legs, "pr.yml's CodeQL legs must equal codeql.yml's full matrix (one configuration per language)") # 9. Every caller GRANTS what codeql.yml requests (a called workflow cannot # request more than its caller holds: startup failure, not a downgrade). # The caller list is DERIVED, never hardcoded: an earlier revision listed # callers by hand and stayed green while a third caller nobody had listed # failed every run at startup. callers = sorted( path.name for path in workflow_dir.glob("*.yml") if "uses: ./.github/workflows/codeql.yml" in path.read_text(encoding="utf-8") ) require({"pr.yml", "dry-run.yml", "release.yml"} <= set(callers), "PR CI, the dry run and the release must call codeql.yml, callers: " + (", ".join(callers) or "none")) for caller_name in callers: call_body = job_body(text(caller_name), "codeql") or "" require("uses: ./.github/workflows/codeql.yml" in call_body, f"{caller_name} must call codeql.yml from a job named codeql") for perm, level in (("security-events", "write"), ("actions", "read"), ("contents", "read")): require(re.search(rf"^ {perm}:\s*{level}\s*$", call_body, re.M) is not None, f"{caller_name} must grant {perm}: {level} to the codeql job") light = re.search(r"(?m)^ light: true$", call_body) is not None require(light == (caller_name == "pr.yml"), f"{caller_name}: only PR CI runs the PR scan (light: true); " "the dry run and the release run the full scan") # 10. PR CI: the codeql job runs for any selected codeql-* lane and ci-ok # needs it; the release publishes nothing without the CodeQL pass. pr_codeql = job_body(pr, "codeql") or "" cond = re.search(r"(?m)^ if: (.*)$", pr_codeql) for lane in ("codeql-cpp", "codeql-js", "codeql-py", "codeql-actions"): require(cond is not None and f"'{lane}'" in cond.group(1), f"pr.yml codeql must run when lane {lane} is selected") require("matrix: ${{ needs.changes.outputs.codeql }}" in pr_codeql, "pr.yml codeql must run the legs the changes job selected") ci_ok = job_body(pr, "ci-ok") or "" require(re.search(r"(?m)^ needs: \[[^\]]*\bcodeql\b[^\]]*\]$", ci_ok) is not None, "ci-ok must need the codeql job") verify = job_body(text("release.yml"), "verify") or "" require(re.search(r"(?m)^ needs: \[[^\]]*\bcodeql\b[^\]]*\]$", verify) is not None, "release.yml verify must need the codeql job") require("needs.codeql.result == 'success'" in verify, "release.yml verify must require needs.codeql.result == 'success'") if failures: print("Security gate fail-closed contract FAILED:") for failure in failures: print(f" - {failure}") sys.exit(1) print("Security gate fail-closed contract passed") PY