build: vendor API reference data to speed up site builds (#26182)

API generation introduced in #26048 added roughly five minutes to site
builds: median successful main-branch build-workflow duration rose from
4m 05s to 8m 48s across September 3–25. The pre-build command took 4m
41s–4m 53s in each of four jobs on the merge commit.

Vendor the generated presentation model in `data/api-reference.json`,
mark it as generated, and let Docker, Netlify, and local Hugo builds
consume it directly. Keep the existing rendered-output checks.

Move generator tests, strict spec validation, and a byte-for-byte
regeneration check into `docker buildx bake validate-api-reference`,
inheriting the shared Docker base stage. The target uses the pinned
Vacuum release binary with Dockerfile ADD checksum verification and
archive extraction instead of compiling Vacuum from source. The existing
build workflow runs this target alongside the other validation matrix
entries. Only this matrix entry is advisory; its failures do not block
site builds or deployment.

Validated locally with the Docker target, rejection of deliberately
stale data, repeatable generation across directories, a production Hugo
build and verification of 453 API HTML/Markdown page pairs, workflow
syntax checks, and scoped documentation lint. The prebuilt-binary CI
check passed in 1m 40s, down from 5m 15s with source compilation. Remote
layer-cache settings were subsequently removed; local Go cache mounts
remain.

Generated by Codex

.gitattributes

@@ -11,3 +11,5 @@ content/**/*.md linguist-detectable
 /_vendor/** linguist-generated=true
 # Generated CLI reference data (vendored from upstream)
 /data/cli/** linguist-generated=true
+# Generated API presentation data
+/data/api-reference.json linguist-generated=true

.github/workflows/build.yml

@@ -72,6 +72,7 @@ jobs:
 
   validate:
     runs-on: ubuntu-24.04
+    continue-on-error: ${{ matrix.target == 'validate-api-reference' }}
     strategy:
       fail-fast: false
       matrix:
@@ -83,6 +84,7 @@ jobs:
           - test-go-redirects
           - dockerfile-lint
           - validate-vendor
+          - validate-api-reference
     steps:
       -
         name: Checkout

.prettierignore

@@ -1 +1,2 @@
 **/*.md
+/data/api-reference.json

Dockerfile

@@ -8,11 +8,11 @@ ARG VALE_VERSION=3.17.0
 ARG HUGO_VERSION=0.163.0
 ARG NODE_VERSION=24
 ARG PAGEFIND_VERSION=1.5.2
+ARG TARGETARCH
 
 # base defines the generic base stage
 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS base
 RUN apk add --no-cache \
-    bash \
     git \
     nodejs \
     npm \
@@ -42,9 +42,6 @@ WORKDIR /project
 COPY --from=hugo /out/hugo /bin/hugo
 COPY --from=npm /out/node_modules node_modules
 COPY . .
-RUN --mount=type=cache,target=/root/go/pkg/mod \
-    --mount=type=cache,target=/root/.cache/go-build \
-    ./hack/api-docs/run.sh test && ./hack/api-docs/run.sh generate
 
 # build creates production builds with Hugo
 FROM build-base AS build
@@ -70,6 +67,37 @@ RUN node hack/api-docs/verify-output.mjs public
 FROM ghcr.io/rvben/rumdl:0.2.49-alpine AS lint
 RUN --mount=type=bind,target=. rumdl check content
 
+# vacuum downloads the prebuilt validator for the target architecture
+FROM scratch AS vacuum-amd64
+ADD --unpack --checksum=sha256:973b8ed30cd36533da4cdb76729e833324a3043bad48c74f61b6a1cc3df40b78 \
+    https://github.com/daveshanley/vacuum/releases/download/v0.30.3/vacuum_0.30.3_linux_x86_64.tar.gz /out/
+
+FROM scratch AS vacuum-arm64
+ADD --unpack --checksum=sha256:04b604df0b1b570d5abd58b577c816122da162eb8ba12a7f70c3060aec12b3a9 \
+    https://github.com/daveshanley/vacuum/releases/download/v0.30.3/vacuum_0.30.3_linux_arm64.tar.gz /out/
+
+FROM vacuum-${TARGETARCH} AS vacuum
+
+# validate-api-reference checks the vendored presentation data
+FROM base AS validate-api-reference
+RUN apk add --no-cache bash
+WORKDIR /project
+COPY --from=vacuum /out/vacuum /usr/local/bin/vacuum
+COPY hack/api-docs ./hack/api-docs
+COPY content/reference/api ./content/reference/api
+COPY data/api-reference.json ./data/api-reference.json
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build <<"EOT"
+set -eu
+cp data/api-reference.json /tmp/api-reference-committed.json
+./hack/api-docs/run.sh test
+./hack/api-docs/run.sh generate
+if ! cmp -s /tmp/api-reference-committed.json data/api-reference.json; then
+  echo >&2 'ERROR: API reference data is stale. Run ./hack/api-docs/run.sh generate and commit data/api-reference.json.'
+  exit 1
+fi
+EOT
+
 # test validates HTML output and checks for broken links
 FROM wjdp/htmltest:v${HTMLTEST_VERSION} AS test
 WORKDIR /test

compose.yaml

@@ -11,19 +11,3 @@ services:
         - action: sync
           path: .
           target: /project
-          ignore:
-            - hack/api-docs/
-            - content/reference/api/hub/latest.yaml
-            - content/reference/api/dvp/latest.yaml
-            - content/reference/api/registry/latest.yaml
-            - content/reference/api/ai-governance/api.yaml
-        - action: rebuild
-          path: hack/api-docs
-        - action: rebuild
-          path: content/reference/api/hub/latest.yaml
-        - action: rebuild
-          path: content/reference/api/dvp/latest.yaml
-        - action: rebuild
-          path: content/reference/api/registry/latest.yaml
-        - action: rebuild
-          path: content/reference/api/ai-governance/api.yaml

content/reference/api/_content.gotmpl

@@ -1,6 +1,6 @@
 {{- $data := index hugo.Data "api-reference" -}}
 {{- if not $data }}
-  {{ errorf "API reference data missing: run ./hack/api-docs/run.sh generate before Hugo" }}
+  {{ errorf "API reference data missing: regenerate data/api-reference.json with ./hack/api-docs/run.sh generate" }}
 {{ end -}}
 {{- if ne (int $data.modelVersion) 1 }}
   {{ errorf "Unsupported API presentation model version" }}

docker-bake.hcl

@@ -40,7 +40,7 @@ target "release" {
 }
 
 group "validate" {
-  targets = ["lint", "test", "unused-media", "test-go-redirects", "dockerfile-lint", "validate-vendor"]
+  targets = ["lint", "test", "unused-media", "test-go-redirects", "dockerfile-lint", "validate-vendor", "validate-api-reference"]
 }
 
 target "test" {
@@ -169,6 +169,12 @@ target "validate-vendor" {
   output = ["type=cacheonly"]
 }
 
+target "validate-api-reference" {
+  target = "validate-api-reference"
+  output = ["type=cacheonly"]
+  provenance = false
+}
+
 variable "UPSTREAM_MODULE_NAME" {
   default = null
 }
@@ -188,4 +194,4 @@ target "validate-upstream" {
   target = "validate-upstream"
   output = ["type=cacheonly"]
   provenance = false
-}
+}

hack/api-docs/README.md

@@ -1,31 +1,41 @@
 # API reference build
 
-This tool validates the Hub, DVP, Registry, and AI Governance OpenAPI sources
+This tool validates the Hub, DVP, Registry, AI Governance, and Sandboxes OpenAPI sources
 in `content/reference/api/` and generates presentation data for Hugo. Engine
 retains its existing sources and renderer.
 
 ## Commands
 
-Use the Go version declared in `go.mod`, Hugo, and the site's Node dependencies.
-From the repository root:
+Site builds use the committed `data/api-reference.json`. To build the site,
+use Hugo and the site's Node dependencies. From the repository root:
 
 ```console
 $ npm ci
-$ ./hack/api-docs/run.sh test
-$ ./hack/api-docs/run.sh generate
 $ hugo server
 ```
 
 For a static build with HTML/Markdown checks, run
 `./hack/api-docs/run.sh build`. To serve that build on port 1314, run
 `./hack/api-docs/run.sh serve`. Set `DOCS_URL` and `DOCS_PORT` when using another
-address. Run generation again after changing a specification or source manifest entry.
-With `docker compose watch`, changes to the registered specifications or
-`hack/api-docs/` rebuild the server image and regenerate the reference.
+address. These commands use the committed data without running the generator.
 
-Docker builds and Netlify deploy previews run generation before Hugo. Generated
-data, validation reports, binaries, and local builds go under `tmp/api-reference/`.
-Hugo reports an error if the generated data is absent.
+After changing a specification, source manifest entry, or generator code, use
+the Go version declared in `hack/api-docs/go.mod` to regenerate the data.
+
+Install the Vacuum version pinned in the [Dockerfile](../../Dockerfile) and make
+the `vacuum` executable available on `PATH`. The script runs Vacuum and the Go
+generator without installing tools.
+
+```console
+$ ./hack/api-docs/run.sh test
+$ ./hack/api-docs/run.sh generate
+```
+
+Commit `data/api-reference.json` with the source changes. With
+`docker compose watch`, the regenerated JSON syncs to the server and Hugo
+rebuilds the reference. Docker builds and Netlify deploy previews also use the
+committed JSON. Validation reports and local builds remain under
+`tmp/api-reference/`.
 
 ## Processor inputs
 
@@ -56,9 +66,7 @@ relative to each original file, then moves `index.md` files to flattened paths.
 It preserves code examples and other text. API pages use the same processing as
 other pages; links that already use published URLs remain unchanged.
 
-The published YAML URLs still serve the source files directly. There is no
-conversion step, snapshot dependency, Node migration package, or source archive
-in the build.
+The published YAML URLs still serve the source files directly.
 
 ## Validation
 
@@ -68,10 +76,25 @@ in the build.
 $ ./hack/api-docs/run.sh check
 ```
 
-Generation runs the same strict validation. Any diagnostic fails the build;
+Generation runs the same strict validation. Any diagnostic fails generation;
 there is no exception baseline. Reports are written to
 `tmp/api-reference/validation.json`.
 
+The build workflow's validation matrix runs API reference checks alongside the
+other validation targets. It runs tests, validates the sources, regenerates the
+JSON, and compares the result with the committed file to detect stale data.
+This matrix entry is advisory: failures do not block site builds or deployment.
+
+Run the same check locally with Docker:
+
+```console
+$ docker buildx bake validate-api-reference
+```
+
+This target runs generator tests, strict validation, and a byte-for-byte
+comparison inside Docker. It leaves the working tree unchanged and is also
+included in the `validate` Bake group.
+
 ## Tests and scope
 
 Go fixtures cover dialects, references, recursion, boolean schemas, examples,

hack/api-docs/main.go

@@ -749,7 +749,7 @@ func run() error {
 		return fmt.Errorf("%d blocking diagnostics; see tmp/api-reference/validation.json", blocking)
 	}
 	if command == "generate" {
-		if e = writeJSON(filepath.Join(out, "data/api-reference.json"), Object{"modelVersion": 1, "apis": models, "legacyAPIs": manifest.LegacyAPIs}); e != nil {
+		if e = writeJSON(filepath.Join(root, "data/api-reference.json"), Object{"modelVersion": 1, "apis": models, "legacyAPIs": manifest.LegacyAPIs}); e != nil {
 			return e
 		}
 	}

hack/api-docs/run.sh

@@ -2,37 +2,24 @@
 set -euo pipefail
 ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
 cd "$ROOT"
-BIN="$ROOT/tmp/api-reference/bin"
-mkdir -p "$BIN" "$ROOT/tmp/api-reference/reports"
+mkdir -p "$ROOT/tmp/api-reference/reports"
 export GOWORK=off
-bootstrap() {
-  (cd hack/api-docs && go build -o "$BIN/api-docs" .)
-  if [[ ! -x "$BIN/vacuum-v0.30.3" ]]; then
-    GOBIN="$BIN" go install github.com/daveshanley/vacuum@v0.30.3
-    mv "$BIN/vacuum" "$BIN/vacuum-v0.30.3"
-  fi
+api_docs() {
+  (cd hack/api-docs && go run . "$@" "$ROOT")
 }
 policy() {
   local sources
-  sources=$("$BIN/api-docs" sources "$ROOT")
+  sources=$(api_docs sources)
   while IFS=$'\t' read -r api source; do
-    "$BIN/vacuum-v0.30.3" lint --no-update-check --remote=false --ruleset hack/api-docs/validation/rules.yaml --fail-severity error --min-score 0 --no-banner --no-style --details "$source" > "tmp/api-reference/reports/$api-vacuum.txt" 2>&1 || {
+    vacuum lint --no-update-check --remote=false --ruleset hack/api-docs/validation/rules.yaml --fail-severity error --min-score 0 --no-banner --no-style --details "$source" > "tmp/api-reference/reports/$api-vacuum.txt" 2>&1 || {
       cat "tmp/api-reference/reports/$api-vacuum.txt"; return 1;
     }
   done <<< "$sources"
 }
-generate() {
-  bootstrap
-  policy
-  "$BIN/api-docs" generate "$ROOT"
-}
 case "${1:-build}" in
-  bootstrap) bootstrap ;;
-  check) bootstrap; policy; "$BIN/api-docs" check "$ROOT" ;;
-  generate) generate ;;
+  check|generate) policy; api_docs "$1" ;;
   test) (cd hack/api-docs && go test ./...) ;;
   build|serve)
-    generate
     hugo --destination tmp/api-reference/site --baseURL "${DOCS_URL:-http://localhost:1314}" --cleanDestinationDir
     node hack/flatten-and-resolve.js tmp/api-reference/site
     node hack/api-docs/verify-output.mjs tmp/api-reference/site
@@ -40,5 +27,5 @@ case "${1:-build}" in
       exec python3 -m http.server "${DOCS_PORT:-1314}" --bind 127.0.0.1 --directory tmp/api-reference/site
     fi
     ;;
-  *) printf '%s\n' 'Usage: run.sh bootstrap|check|generate|test|build|serve' >&2; exit 2 ;;
+  *) printf '%s\n' 'Usage: run.sh check|generate|test|build|serve' >&2; exit 2 ;;
 esac

hack/api-docs/verify-output.mjs

@@ -4,7 +4,7 @@ import { isDeepStrictEqual } from "node:util";
 const root = path.resolve(import.meta.dirname, "../..");
 const base = path.resolve(process.argv[2] || "public");
 const data = JSON.parse(
-  fs.readFileSync(path.join(root, "tmp/api-reference/data/api-reference.json")),
+  fs.readFileSync(path.join(root, "data/api-reference.json")),
 );
 const decode = (s) =>
   s
@@ -157,6 +157,7 @@ const bytes = files(path.join(base, "reference/api")).reduce(
   (n, p) => n + fs.statSync(p).size,
   0,
 );
+fs.mkdirSync(path.join(root, "tmp/api-reference/reports"), { recursive: true });
 fs.writeFileSync(
   path.join(root, "tmp/api-reference/reports/output.json"),
   JSON.stringify(

hack/sandboxes/README.md

@@ -34,7 +34,8 @@ installation page.
 The specification is registered in `hack/api-docs/sources.json`. The shared API
 renderer generates an overview and separate operation and schema pages under
 `/reference/api/sandboxes/latest/`, in HTML and Markdown. Run
-`./hack/api-docs/run.sh generate` before Hugo. Strict validation checks the
+`./hack/api-docs/run.sh generate` after importing a specification and commit
+`data/api-reference.json` with the source changes. Strict validation checks the
 specification and its examples. Fix failures upstream and re-import the export.
 
 To update, change both commit IDs in `source.json`, review additions or removals

hugo.yaml

@@ -329,8 +329,6 @@ module:
   mounts:
     - source: data
       target: data
-    - source: tmp/api-reference/data
-      target: data
     # Mount the assets directory so it doesn't get overwritten
     - source: assets
       target: assets

netlify.toml

@@ -3,7 +3,6 @@ publish = "public"
 
 [context.deploy-preview.environment]
 NODE_VERSION = "24"
-GO_VERSION = "1.26.5"
 NODE_ENV = "production"
 HUGO_VERSION = "0.163.0"
 HUGO_ENABLEGITINFO = "true"
@@ -11,4 +10,4 @@ HUGO_ENVIRONMENT = "preview"
 SECRETS_SCAN_OMIT_PATHS = "public/contribute/file-conventions/index.html"
 
 [context.deploy-preview]
-command = "./hack/api-docs/run.sh generate && hugo --gc --minify -b $DEPLOY_PRIME_URL && node hack/flatten-and-resolve.js && node hack/api-docs/verify-output.mjs && npx pagefind@v1.5.2"
+command = "hugo --gc --minify -b $DEPLOY_PRIME_URL && node hack/flatten-and-resolve.js && node hack/api-docs/verify-output.mjs && npx pagefind@v1.5.2"