ci: run the Windows tests on a runner with KVM (#1369)

Run the Windows tests on `ubuntu-latest-kvm` as a PR job, with the installed Windows cached, minutes instead of ~1h40m. Aligned with https://gitea.com/gitea/gitea-mcp/pulls/321 and https://gitea.com/gitea/cli/pulls/79.

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1369
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
Co-authored-by: Zettat123 <zettat123@gmail.com>

.gitea/workflows/test-windows.yml

@@ -1,21 +0,0 @@
-name: windows
-on:
-  schedule:
-    - cron: '0 3 * * 1'
-  workflow_dispatch:
-
-env:
-  DOCKER_CONFIG: /tmp/docker-noauth # see test.yml
-
-jobs:
-  test:
-    name: test on windows
-    runs-on: ubuntu-latest
-    timeout-minutes: 180
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: ./.gitea/actions/go-setup
-      - name: prepare anonymous docker config
-        run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
-      - name: test on windows
-        run: make test-windows

.gitea/workflows/test.yml

@@ -57,6 +57,29 @@ jobs:
           make coverage-report
           cat .tmp/coverage.md >> "$GITHUB_STEP_SUMMARY"
 
+  windows:
+    name: test on windows
+    runs-on: ubuntu-latest-kvm
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - uses: ./.gitea/actions/go-setup
+      - name: prepare anonymous docker config
+        run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
+      - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/windows-vm
+          key: windows-vm
+      - name: test on windows
+        id: test-windows
+        run: WINDOWS_TEST_CACHE="$HOME/.cache/windows-vm" make test-windows
+        env:
+          WINDOWS_TEST_TIMEOUT: 45
+      - if: always() && steps.test-windows.outputs.save-cache == 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/windows-vm
+          key: windows-vm
+
   e2e:
     name: gitea compatibility (${{ matrix.gitea_image }})
     runs-on: ubuntu-latest

DEVELOPMENT.md

@@ -77,4 +77,5 @@ repository runners. A run path is `<workflow>@<ref>` and a log row is `<timestam
 The host must be amd64 and allow privileged containers. `WINDOWS_TEST_TIMEOUT` sets the minutes
 allowed for installing Windows and running the tests, 165 by default. The VM uses KVM when the host
 has `/dev/kvm`, otherwise Windows is emulated and much slower. Docker-backed tests skip in the VM
-and the race detector is off, it needs cgo. CI runs this weekly and on demand.
+and the race detector is off, it needs cgo. `WINDOWS_TEST_CACHE` names a directory to keep the
+installed Windows in, so later runs skip the install. CI runs this on every pull request.

Makefile

@@ -153,7 +153,7 @@ test-dind: ## run the daemon-facing tests against the built dind image or podman
 
 .PHONY: test-windows
 test-windows: ## run the tests on Windows Server Core in a QEMU VM on the docker host
-	@WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
+	@GO=$(GO) WINDOWS_TEST_IMAGE=$(WINDOWS_TEST_IMAGE) ./scripts/test-windows.sh
 
 E2E_CONCURRENCY ?= 8
 

scripts/test-windows.ps1

@@ -5,7 +5,7 @@ $root = $PSScriptRoot
 try {
     New-Item -ItemType Directory "$root\logs" | Out-Null
     $env:Path = "C:\git\cmd;C:\git\bin;C:\pwsh;C:\node;$env:Path"
-    git config --global --add safe.directory C:/OEM/src # setup copied it as SYSTEM, the tests run as the logged-on user
+    git config --global --add safe.directory C:/tests/src
 
     $failed = 0
     Write-Output "==> Running $((Get-Content "$root\tests.txt").Count) test binaries on $([Environment]::ProcessorCount) cores"
@@ -13,7 +13,7 @@ try {
         $binary, $package = $_ -split ' ', 2
         $started = Get-Date
         Set-Location "$using:root\src\$package"
-        cmd.exe /d /c "$using:root\bin\$binary -test.v -test.timeout=60m -test.parallel=8 > $using:root\logs\$($package -replace '/', '-').log 2>&1"
+        cmd.exe /d /c "$using:root\bin\$binary -test.v -test.paniconexit0 -test.timeout=60m -test.parallel=8 > $using:root\logs\$($package -replace '/', '-').log 2>&1"
         [pscustomobject]@{ Package = $package; ExitCode = $LASTEXITCODE; Seconds = [int]((Get-Date) - $started).TotalSeconds }
     } | ForEach-Object {
         $line = "ok $($_.Package) $($_.Seconds)s"

scripts/test-windows.sh

@@ -3,11 +3,15 @@ set -euo pipefail
 
 image="${WINDOWS_TEST_IMAGE:?}"
 timeout="${WINDOWS_TEST_TIMEOUT:-165}"
-name="gitea-runner-windows-test-$$"
+cache="${WINDOWS_TEST_CACHE:-}" # keeps the installed Windows between runs
+go="${GO:-go}"
 work="$(mktemp -d)"
+volume="gitea-runner-windows-test-${work##*.}"
+name=""
 
 cleanup() {
   docker rm -fv "$name" >/dev/null 2>&1 || true
+  docker volume rm "$volume" >/dev/null 2>&1 || true
   rm -rf "$work"
 }
 trap cleanup EXIT
@@ -19,21 +23,42 @@ fi
 
 echo "==> Building Windows test binaries"
 oem="$work/oem"
-mkdir -p "$oem/bin" "$oem/src" "$work/shared"
+shared="$work/shared"
-module="$(go list -m)"
+payload="$work/payload"
-packages="$(GOOS=windows go list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./...)"
+src="$payload/src"
-GOOS=windows GOARCH=amd64 go test -c -ldflags='-s -w' -o "$oem/bin/" $packages
+mkdir -p "$oem" "$shared" "$payload/bin" "$src"
+module="$("$go" list -m)"
+packages="$(GOOS=windows "$go" list -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' ./...)"
+GOOS=windows GOARCH=amd64 "$go" test -c -ldflags='-s -w' -o "$payload/bin/" $packages
 for package in $packages; do
   echo "${package##*/}.test.exe ${package#"$module"/}"
-done > "$oem/tests.txt"
+done > "$payload/tests.txt"
 git ls-files -z --cached --others --exclude-standard | while IFS= read -r -d '' file; do
   if [ -e "$file" ]; then printf '%s\0' "$file"; fi
-done | tar --null -T - -cf - | tar -xf - -C "$oem/src"
+done | tar --null -T - -cf - | tar -xf - -C "$src"
-git -C "$oem/src" init -q -b main # a repository like the CI checkout, for the tests that read the github context from it
+git -C "$src" init -q -b main # a repository like the CI checkout, for the tests that read the github context from it
-git -C "$oem/src" add -Af .
+git -C "$src" add -Af .
-git -C "$oem/src" -c user.name=test -c user.email=test@localhost -c commit.gpgsign=false -c core.hooksPath=/dev/null commit -qm test
+git -C "$src" -c user.name=test -c user.email=test@localhost -c commit.gpgsign=false -c core.hooksPath=/dev/null commit -qm test
-git -C "$oem/src" remote add origin https://gitea.com/gitea/runner
+git -C "$src" remote add origin https://gitea.com/gitea/runner
-cp scripts/test-windows-setup.ps1 scripts/test-windows.ps1 "$oem/"
+cp scripts/test-windows.ps1 "$payload/"
+tar -cf "$shared/payload.tar" -C "$payload" .
+cp scripts/test-windows-setup.ps1 "$oem/"
+sed 's/$/\r/' > "$shared/run.bat" <<'EOF'
+@echo off
+set log=\\host.lan\Data\install.log
+rd /s /q C:\tests 2> nul
+md C:\tests
+tar.exe -xf \\host.lan\Data\payload.tar -C C:\tests >> %log% 2>&1
+C:\pwsh\pwsh.exe -NoProfile -ExecutionPolicy Bypass -File C:\tests\test-windows.ps1 >> %log% 2>&1
+if not exist \\host.lan\Data\exitcode (echo 1)> \\host.lan\Data\exitcode
+cd /d C:\
+rd /s /q C:\tests
+if not exist \\host.lan\Data\save (shutdown.exe /s /t 600 & exit /b)
+:collected
+if not exist \\host.lan\Data\collected (ping.exe -n 2 127.0.0.1 > nul & goto collected)
+defrag.exe C: /L > nul
+shutdown.exe /s /t 0
+EOF
 sed 's/$/\r/' > "$oem/install.bat" <<'EOF' # a policy turns off real-time scanning, AMSI blocks scripts that do it
 @echo off
 set log=\\host.lan\Data\install.log
@@ -41,37 +66,84 @@ echo ==^> install.bat started at %TIME% >> %log%
 reg.exe add "HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\Real-Time Protection" /v DisableRealtimeMonitoring /t REG_DWORD /d 1 /f > nul
 powershell.exe -NoProfile -ExecutionPolicy Bypass -File C:\OEM\test-windows-setup.ps1 >> %log% 2>&1
 echo ==^> setup exited with %ERRORLEVEL% >> %log%
-if %ERRORLEVEL% equ 0 C:\pwsh\pwsh.exe -NoProfile -ExecutionPolicy Bypass -File C:\OEM\test-windows.ps1 >> %log% 2>&1
+if %ERRORLEVEL% neq 0 (shutdown.exe /s /t 600 & exit /b)
-if not exist \\host.lan\Data\exitcode (echo 1)> \\host.lan\Data\exitcode
+(
-shutdown.exe /s /t 600
+  echo @echo off
+  echo :wait
+  echo if exist \\host.lan\Data\run.bat goto run
+  echo ping.exe -n 3 127.0.0.1 ^> nul
+  echo goto wait
+  echo :run
+  echo call \\host.lan\Data\run.bat
+)> C:\run.bat
+schtasks.exe /create /tn tests /sc onlogon /it /rl highest /tr C:\run.bat /f > nul
+call C:\run.bat
 EOF
 
-kvm=Y cores="$(nproc)"
+kvm=Y cores=max
 if ! docker run --rm --privileged --entrypoint sh "$image" -c 'test -w /dev/kvm'; then
   kvm=N cores=4 # more emulated vCPUs slow Windows down
   echo "==> The docker host has no usable /dev/kvm, Windows runs emulated and much slower"
 fi
 
-echo "==> Installing Windows"
+vm=(-e VERSION=2025 -e EDITION=core -e RAM_SIZE=8G -e DISK_SIZE=32G -e DISK_FMT=qcow2)
-docker create --rm --name "$name" --privileged -e VERSION=2025 -e EDITION=core -e KVM="$kvm" -e CPU_CORES="$cores" -e RAM_SIZE=8G \
+fingerprint="$(printf '%s\n' "$image" "${vm[*]}" | cat - "$oem/install.bat" "$oem/test-windows-setup.ps1" | sha256sum | cut -d' ' -f1)" # what the installed disk depends on
-  -e DISK_SIZE=32G -e DISK_CACHE=unsafe -e DISK_IO=threads --entrypoint timeout "$image" "$((timeout + 10))m" /usr/bin/tini -s /run/entry.sh >/dev/null # the timeout stops the VM even when this script is killed before its cleanup runs
+docker volume create "$volume" >/dev/null
-docker cp "$oem" "$name:/oem"
+cached=N
-docker cp "$work/shared" "$name:/shared"
+if [ -n "$cache" ] && [ "$(cat "$cache/fingerprint" 2>/dev/null)" = "$fingerprint" ]; then
-docker start "$name" >/dev/null
+  cached=Y
-docker logs -f "$name" &
+  echo "==> Booting the cached Windows install"
-while sleep 10 && docker container inspect "$name" >/dev/null 2>&1; do
+else
-  if ((SECONDS % 300 < 10)); then
+  echo "==> Installing Windows"
-    echo "==> $((SECONDS / 60))m elapsed, VM disk $(docker exec "$name" du -sh /storage | cut -f1)"
+  if [ -n "$cache" ]; then
+    touch "$shared/save"
   fi
-done &
-docker exec "$name" sh -c 'cd /shared && touch install.log summary.txt && tail -qf install.log summary.txt' &
-
-if ! timeout "${timeout}m" docker exec "$name" sh -c 'until [ -f /shared/exitcode ]; do sleep 10; done'; then
-  echo "Windows stopped or did not finish the tests within $timeout minutes" >&2
-  exit 1
 fi
+deadline=$((SECONDS + timeout * 60))
+boot() {
+  name="gitea-runner-windows-test-${work##*.}-$1" # a new name ends the log and monitor loops of the previous VM
+  docker create --rm --name "$name" --privileged -v "$volume:/storage" "${vm[@]}" -e KVM="$kvm" -e CPU_CORES="$cores" -e DISK_CACHE=unsafe -e DISK_IO=threads \
+    --entrypoint timeout "$image" "$(((deadline - SECONDS) / 60 + 10))m" /usr/bin/tini -s /run/entry.sh >/dev/null # the timeout stops the VM even when this script is killed before its cleanup runs
+  if [ "$cached" = Y ]; then
+    docker cp "$cache/storage/." "$name:/storage"
+  else
+    docker cp "$oem" "$name:/oem"
+  fi
+  docker cp "$shared" "$name:/shared"
+  docker start "$name" >/dev/null
+  docker logs -f "$name" &
+  while sleep 10 && docker container inspect "$name" >/dev/null 2>&1; do
+    if ((SECONDS % 300 < 10)); then
+      echo "==> $((SECONDS / 60))m elapsed, VM disk $(docker exec "$name" du -sh /storage | cut -f1)"
+    fi
+  done &
+  docker exec "$name" sh -c 'cd /shared && touch install.log summary.txt && tail -qf install.log summary.txt' &
+}
+
+boots=1
+boot "$boots"
+until docker exec "$name" test -f /shared/exitcode 2>/dev/null; do
+  if ((SECONDS >= deadline)) || [ "$(docker container inspect -f '{{.State.Running}}' "$name" 2>/dev/null)" != true ]; then
+    echo "Windows stopped or did not finish the tests within $timeout minutes" >&2
+    exit 1
+  fi
+  if docker logs "$name" 2>&1 | grep -q 'No bootable option or device was found'; then # the firmware can give up on the install image
+    if ((boots == 3)); then
+      echo "Windows did not boot from its install image $boots times" >&2
+      exit 1
+    fi
+    echo "==> Windows did not boot from its install image, starting a new VM"
+    docker rm -fv "$name" >/dev/null
+    docker volume rm "$volume" >/dev/null && docker volume create "$volume" >/dev/null
+    boots=$((boots + 1))
+    boot "$boots"
+  fi
+  sleep 10
+done
 results="$work/results"
+docker exec "$name" rm /shared/payload.tar
 docker cp "$name:/shared" "$results"
+docker exec "$name" touch /shared/collected
 
 tr -d '\r' < "$results/summary.txt" | while read -r result package _; do
   if [ "$result" = FAIL ]; then
@@ -87,4 +159,21 @@ echo "==> Slowest tests"
 find "$results" -name '*.log' -exec cat {} + | tr -d '\r' | sed -nE 's/^--- (PASS|FAIL): (\S+) \(([0-9.]+)s\)$/\3 \2/p' | sort -rn | awk 'NR <= 20'
 echo "==> Packages"
 tr -d '\r' < "$results/summary.txt"
+
+if [ -f "$shared/save" ]; then
+  echo "==> Saving the Windows install to $cache"
+  until [ "$(docker container inspect -f '{{.State.Running}}' "$name" 2>/dev/null)" != true ]; do
+    sleep 2
+  done
+  docker run --rm -v "$volume:/storage" --entrypoint sh "$image" -c \
+    'cd /storage && rm -f -- *.iso && qemu-img convert -O qcow2 data.qcow2 data.new && mv data.new data.qcow2'
+  rm -rf "$cache" && mkdir -p "$cache"
+  helper="$(docker create -v "$volume:/storage" --entrypoint true "$image")"
+  docker cp "$helper:/storage" "$cache/storage"
+  docker rm "$helper" >/dev/null
+  echo "$fingerprint" > "$cache/fingerprint"
+  if [ -n "${GITHUB_OUTPUT:-}" ]; then
+    echo "save-cache=true" >> "$GITHUB_OUTPUT"
+  fi
+fi
 exit "$(tr -d '\r\n' < "$results/exitcode")"