#!/usr/bin/env bash
# The code-scanning gate must FAIL CLOSED -- and CodeQL must run where it gates.
#
# History. `codeql-gate` in .github/workflows/_security.yml (since moved into
# .github/workflows/codeql.yml as the analyze job's alert check) counted open
# alerts with:
#
# ALERTS=$(gh api '.../code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0")
#
# That reads as defensive and is the exact opposite. The job declared no
# `permissions:` block, so it inherited the workflow's `contents: read` and the
# alert API answered 403. `gh` writes the API error BODY to stdout, so `ALERTS`
# became the string
#
# {"message":"Resource not accessible by integration",...,"status":"403"}0
#
# every `[ ... -gt ... ]` then failed as a non-integer comparison, the `if` took
# its false branch, and the step printed "CodeQL gate passed (0 alerts)" and
# exited 0. Observed live in a GREEN run (34700514102 / job 103571322660), while
# the repository had an open high-severity alert.
#
# A security gate that cannot read its input must stop the build, not wave it
# through. This test pins both halves of that: the permission that makes the
# read possible, and the fail-closed handling for when it still is not.
#
# It also pins where CodeQL runs. The former PR gate polled up to 150 min per
# PR for a separate CodeQL run and starved PR CI of runners; now:
# - no pull_request trigger and no job that polls for a CodeQL run;
# - PR CI calls codeql.yml as its own job (needed by ci-ok), one leg per
# selected codeql-* lane; each leg analyses its language's whole
# first-party scope (never narrowed by the diff); c-cpp compiles the
# vendored objects before tracing, then traces the first-party build plus
# the vendored files the PR changes;
# - push to main, the dry run and the release run the full scan, and the
# release's verify needs it;
# - superseded push and PR runs are cancelled, dispatched ones never.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
python3 - "$ROOT" <<'PY'
from __future__ import annotations
import json
import pathlib
import re
import sys
root = pathlib.Path(sys.argv[1])
failures: list[str] = []
def require(condition: bool, message: str) -> None:
if not condition:
failures.append(message)
workflow_dir = root / ".github" / "workflows"
def text(name: str) -> str:
return (workflow_dir / name).read_text(encoding="utf-8")
def job_body(source: str, job: str) -> str | None:
m = re.search(rf"\n {re.escape(job)}:\n(?P
(?: .*\n|\n)*)", source)
return m.group("body") if m else None
def code_only(source: str) -> str:
return "\n".join(line for line in source.splitlines() if not line.lstrip().startswith("#"))
def steps(body: str) -> list[tuple[str, str]]:
"""(name, step text) per `- name:`/`- uses:` step of a job body."""
found = []
for m in re.finditer(r"(?ms)^ - (?P.*?)(?=^ - |\Z)", body):
step = m.group("step")
name = re.search(r"(?m)^(?:name: |\s+name: )(.*)$", step)
found.append((name.group(1).strip() if name else step.splitlines()[0], step))
return found
workflow = text("codeql.yml")
gate = job_body(workflow, "analyze")
require(gate is not None, "codeql.yml no longer defines an analyze job")
body = gate or ""
# 1. The read has to be possible at all.
require(
re.search(r"^ permissions:\s*$", body, re.M) is not None,
"analyze must declare its own permissions block -- inheriting the "
"workflow default leaves it without security-events access and the alert "
"API answers 403",
)
require(
re.search(r"^ security-events:\s*(read|write)\s*$", body, re.M) is not None,
"analyze must hold security-events (write implies read), or the "
"code-scanning alert query 403s and the gate cannot see any alert",
)
# 2. The failure direction. `|| echo "0"` on the alert query is the specific
# construct that turned an API error into a clean pass.
require(
re.search(r"code-scanning/alerts[^\n]*\|\|\s*echo", body) is None,
"the alert count must not fall back to a literal on error -- an unreadable "
"alert list is not zero alerts",
)
require(
"BLOCKED: cannot read code scanning alerts" in body,
"the alert check must fail with a named error when the alert API cannot be read",
)
require(
re.search(r"alert count was not a number", body) is not None,
"the alert check must reject a non-numeric alert count instead of comparing it",
)
# 3. The count is produced in a command substitution, so a bare `exit` inside
# the helper would end only the subshell and let the caller proceed with an
# empty value -- the same class of silent pass. The callers must check.
assignments = re.findall(r"^\s*ALERTS2?=\$\(count_open_alerts\)(.*)$", body, re.M)
require(
len(assignments) >= 2,
"expected both alert samples to go through the checked helper",
)
require(
all("|| exit 1" in tail for tail in assignments),
"every count_open_alerts call must be followed by || exit 1 -- a failure "
"inside a command substitution does not stop the caller on its own",
)
# 4. Triggers: never per pull request (PR CI calls the workflow instead), on
# push to main, callable by PR CI, the dry run and the release.
on = re.search(r"(?ms)^on:\n(?P.*?)(?=^\S)", workflow)
triggers = set(re.findall(r"(?m)^ ([a-z_]+):", on.group("body"))) if on else set()
require("pull_request" not in triggers and "pull_request_target" not in triggers,
"codeql.yml must not trigger on pull requests (pr.yml calls it as a job)")
require({"push", "workflow_call"} <= triggers,
f"codeql.yml must run on push to main and be callable, has {sorted(triggers)}")
require(re.search(r"(?m)^ push:\n branches: \[main\]$", workflow) is not None,
"codeql.yml's push trigger must be main")
# 5. Nothing waits for a separate CodeQL run, and _security.yml has no CodeQL.
for path in sorted(workflow_dir.glob("*.yml")):
require("workflows/codeql.yml/runs" not in code_only(path.read_text(encoding="utf-8")),
f"{path.name} polls for a CodeQL run -- CodeQL runs as a job of the caller")
require("codeql" not in code_only(text("_security.yml")).lower(),
"_security.yml must not run or wait for CodeQL")
# 6. Superseded runs are cancelled (push to main, a PR), dispatched never;
# legs of one run never cancel each other.
concurrency = re.search(r"(?ms)^ concurrency:\n(.*?)(?=^ \S)", body)
conc = concurrency.group(1) if concurrency else ""
require("matrix.language" in conc, "the concurrency group must be per language")
require("github.event_name == 'push' && github.ref" in conc,
"push runs must share a ref-scoped group (only the newest main commit is scanned)")
require("github.event.pull_request.number" in conc,
"PR scans must share a per-PR group")
require("cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}" in conc,
"only push and PR scans may be cancelled -- a dry run or release tests one commit")
# 7. The PR scan: analysis never narrowed by the diff; c-cpp prebuilds the
# vendored objects untraced and traces first-party + changed vendored.
names = [name for name, _ in steps(body)]
by_name = dict(steps(body))
def index(fragment: str) -> int:
hits = [i for i, n in enumerate(names) if fragment in n]
return hits[0] if hits else -1
init, analyze = index("Initialize CodeQL"), index("Perform CodeQL Analysis")
prebuild, retrace = index("Build third-party objects"), index("Vendored files this PR changes")
light_build, full_build = index("PR scan, first-party"), index("Build for CodeQL analysis (full)")
require(min(init, analyze, prebuild, retrace, light_build, full_build) >= 0,
f"codeql.yml analyze lost a step, has {names}")
require(-1 < prebuild < init and -1 < retrace < init,
"the vendored prebuild and the retrace list must run BEFORE Initialize CodeQL (untraced)")
require(init < light_build < analyze and init < full_build < analyze,
"the traced builds must run between Initialize CodeQL and the analysis")
for name in names:
if "Initialize CodeQL" in name or "Perform CodeQL Analysis" in name or "open code scanning alerts" in name:
require(not re.search(r"(?m)^\s+if:", by_name[name]),
f"step '{name}' must never be skipped -- a selected leg always analyses")
light_step = by_name.get(names[light_build], "") if light_build >= 0 else ""
require("scripts/build.sh --reuse-vendored --retrace-from" in light_step,
"the PR c-cpp build must reuse the untraced vendored objects and retrace the changed ones")
require("scripts/ci/codeql-retrace-vendored.sh" in (by_name.get(names[retrace], "") if retrace >= 0 else ""),
"the retrace list must come from scripts/ci/codeql-retrace-vendored.sh")
require("scripts/build.sh --vendored-only" in (by_name.get(names[prebuild], "") if prebuild >= 0 else ""),
"the PR c-cpp prebuild must be scripts/build.sh --vendored-only")
require(re.search(r"(?m)^\s+run: scripts/build\.sh\s*$", by_name.get(names[full_build], "") if full_build >= 0 else "") is not None,
"the full c-cpp scan must trace the whole production build (vendored C included)")
require(re.search(r'(?m)^ CBM_NO_CCACHE: "1"$', body) is not None,
"the CodeQL job must disable the compiler cache -- a cache hit compiles nothing to trace")
require("config-file: ./.github/codeql/codeql-config.yml" in body,
"every scan must use the first-party scope in .github/codeql/codeql-config.yml")
# 8. One configuration per language: the full matrix and PR CI's legs agree.
default = re.search(r"fromJSON\(inputs\.matrix \|\| '(?P\{.*?\})'\)", body)
pr = text("pr.yml")
pr_legs = re.search(r"CODEQL='(?P\[.*?\])'", pr)
full_legs = json.loads(default.group("json"))["include"] if default else []
require({leg.get("language") for leg in full_legs} ==
{"c-cpp", "javascript-typescript", "python", "actions"},
"the full scan must cover c-cpp, javascript-typescript, python and actions")
require(pr_legs is not None and json.loads(pr_legs.group("json")) == full_legs,
"pr.yml's CodeQL legs must equal codeql.yml's full matrix (one configuration per language)")
# 9. Every caller GRANTS what codeql.yml requests (a called workflow cannot
# request more than its caller holds: startup failure, not a downgrade).
# The caller list is DERIVED, never hardcoded: an earlier revision listed
# callers by hand and stayed green while a third caller nobody had listed
# failed every run at startup.
callers = sorted(
path.name
for path in workflow_dir.glob("*.yml")
if "uses: ./.github/workflows/codeql.yml" in path.read_text(encoding="utf-8")
)
require({"pr.yml", "dry-run.yml", "release.yml"} <= set(callers),
"PR CI, the dry run and the release must call codeql.yml, callers: "
+ (", ".join(callers) or "none"))
for caller_name in callers:
call_body = job_body(text(caller_name), "codeql") or ""
require("uses: ./.github/workflows/codeql.yml" in call_body,
f"{caller_name} must call codeql.yml from a job named codeql")
for perm, level in (("security-events", "write"), ("actions", "read"), ("contents", "read")):
require(re.search(rf"^ {perm}:\s*{level}\s*$", call_body, re.M) is not None,
f"{caller_name} must grant {perm}: {level} to the codeql job")
light = re.search(r"(?m)^ light: true$", call_body) is not None
require(light == (caller_name == "pr.yml"),
f"{caller_name}: only PR CI runs the PR scan (light: true); "
"the dry run and the release run the full scan")
# 10. PR CI: the codeql job runs for any selected codeql-* lane and ci-ok
# needs it; the release publishes nothing without the CodeQL pass.
pr_codeql = job_body(pr, "codeql") or ""
cond = re.search(r"(?m)^ if: (.*)$", pr_codeql)
for lane in ("codeql-cpp", "codeql-js", "codeql-py", "codeql-actions"):
require(cond is not None and f"'{lane}'" in cond.group(1),
f"pr.yml codeql must run when lane {lane} is selected")
require("matrix: ${{ needs.changes.outputs.codeql }}" in pr_codeql,
"pr.yml codeql must run the legs the changes job selected")
ci_ok = job_body(pr, "ci-ok") or ""
require(re.search(r"(?m)^ needs: \[[^\]]*\bcodeql\b[^\]]*\]$", ci_ok) is not None,
"ci-ok must need the codeql job")
verify = job_body(text("release.yml"), "verify") or ""
require(re.search(r"(?m)^ needs: \[[^\]]*\bcodeql\b[^\]]*\]$", verify) is not None,
"release.yml verify must need the codeql job")
require("needs.codeql.result == 'success'" in verify,
"release.yml verify must require needs.codeql.result == 'success'")
if failures:
print("Security gate fail-closed contract FAILED:")
for failure in failures:
print(f" - {failure}")
sys.exit(1)
print("Security gate fail-closed contract passed")
PY