From 016c9b716fa53d4a12a1bd25178a98ca72b45a9c Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Thu, 22 Oct 2026 15:50:19 -0500 Subject: [PATCH 1/2] debian --- .github/RUNNER_SETUP.md | 44 ++++++++++++++++++++++++++++++---------- .github/workflows/ci.yml | 14 ++++++------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/.github/RUNNER_SETUP.md b/.github/RUNNER_SETUP.md index 400e3f4..749b812 100644 --- a/.github/RUNNER_SETUP.md +++ b/.github/RUNNER_SETUP.md @@ -1,19 +1,41 @@ -# Self-hosted runner setup (macOS arm64) +# Self-hosted runner setup (Debian / Linux) This repo is public, so only run the runner for trusted workflows. `ci.yml` has no `pull_request` trigger on purpose: a fork PR can edit the workflow and run on this machine. -1. Repo → Settings → Actions → Runners → **New self-hosted runner** → macOS / ARM64. +Jobs use `runs-on: [self-hosted, linux]`, which matches the default labels GitHub gives a Linux runner. +A self-hosted runner has no tool image, so everything the jobs call must be installed on the machine. + +1. Repo → Settings → Actions → Runners → **New self-hosted runner** → Linux (pick x64 or ARM64 to match `uname -m`). Follow the download and `./config.sh --url ... --token ...` steps in `~/actions-runner`. - Keep the default labels (`self-hosted`, `macOS`, `ARM64`). -2. Install as a launchd service: `./svc.sh install && ./svc.sh start`. -3. The service does not load shell profiles. Add the tool directories to `~/actions-runner/.path` - (one `PATH` line, then `./svc.sh stop && ./svc.sh start`): + Keep the default labels (`self-hosted`, `Linux`, plus `X64` or `ARM64`). + Run it as a normal user, not root. +2. Install as a systemd service: `sudo ./svc.sh install && sudo ./svc.sh start` + (`sudo ./svc.sh install ` to run as a specific user). +3. Install the base packages: `sudo apt-get update && sudo apt-get install -y git curl ca-certificates jq tar` +4. Install the tools the workflow calls directly. `uv` provides `uvx ruff` and `uvx pip-audit`, and downloads the Python 3.11 that the hygiene job needs: - ``` - /Users//.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + ```bash + curl -LsSf https://astral.sh/uv/install.sh | sh # installs to ~/.local/bin + + mkdir -p ~/.local/bin && cd "$(mktemp -d)" + case "$(uname -m)" in x86_64) A=x64 ;; aarch64) A=arm64 ;; esac + + curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_${A}.tar.gz" \ + | tar xz gitleaks && install gitleaks ~/.local/bin/ + + curl -sSfL "https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_${A/x64/amd64}.tar.gz" \ + | tar xz actionlint && install actionlint ~/.local/bin/ ``` -4. Install the tools the workflow calls directly: `brew install gitleaks actionlint jq` - (`uv` provides `uvx ruff` and `uvx pip-audit`). -5. Repo → Settings → Actions → General → Fork pull request workflows → + Optional: `sudo apt-get install -y shellcheck`. `actionlint` runs it on `run:` scripts if present. +5. The service does not load shell profiles. Make sure `~/.local/bin` is on the service's PATH by adding this line to + `~/actions-runner/.env`, then `sudo ./svc.sh stop && sudo ./svc.sh start`: + + ``` + PATH=/home//.local/bin:/usr/local/bin:/usr/bin:/bin + ``` + +6. Repo → Settings → Actions → General → Fork pull request workflows → **Require approval for all outside collaborators**. + +Check the setup with `sudo -u env -i PATH=/home//.local/bin:/usr/bin:/bin sh -c 'uvx --version && gitleaks version && actionlint -version && jq --version'`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d303430..f21045c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ env: jobs: lint-notebooks: - runs-on: self-hosted + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v4 with: @@ -27,7 +27,7 @@ jobs: - run: uvx "ruff@${RUFF_VERSION}" check --output-format=github notebooks/ format-notebooks: - runs-on: self-hosted + runs-on: [self-hosted, linux] continue-on-error: true # advisory: notebooks were never formatted steps: - uses: actions/checkout@v4 @@ -36,7 +36,7 @@ jobs: - run: uvx "ruff@${RUFF_VERSION}" format --check notebooks/ secret-scan: - runs-on: self-hosted + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v4 with: @@ -45,7 +45,7 @@ jobs: - run: gitleaks detect --source . --redact --verbose dep-audit: - runs-on: self-hosted + runs-on: [self-hosted, linux] continue-on-error: true # advisory: pins are stale; Dependabot tracks bumps steps: - uses: actions/checkout@v4 @@ -55,7 +55,7 @@ jobs: - run: uvx pip-audit -r requirements.txt --no-deps --disable-pip workflow-lint: - runs-on: self-hosted + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v4 with: @@ -63,7 +63,7 @@ jobs: - run: actionlint notebook-hygiene: - runs-on: self-hosted + runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v4 with: @@ -73,7 +73,7 @@ jobs: summary: if: always() needs: [lint-notebooks, format-notebooks, secret-scan, dep-audit, workflow-lint, notebook-hygiene] - runs-on: self-hosted + runs-on: [self-hosted, linux] steps: - name: Write job summary env: From baf05d43c4568cd3ee8dd4f208913689cf6b74a1 Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Thu, 22 Oct 2026 16:02:05 -0500 Subject: [PATCH 2/2] debian --- .github/RUNNER_SETUP.md | 56 +++++++++++-------------- .github/scripts/setup-tools.sh | 77 ++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 20 ++++++++- 3 files changed, 119 insertions(+), 34 deletions(-) create mode 100644 .github/scripts/setup-tools.sh diff --git a/.github/RUNNER_SETUP.md b/.github/RUNNER_SETUP.md index 749b812..2ae5040 100644 --- a/.github/RUNNER_SETUP.md +++ b/.github/RUNNER_SETUP.md @@ -1,41 +1,33 @@ -# Self-hosted runner setup (Debian / Linux) +# Self-hosted runner setup (Debian container) This repo is public, so only run the runner for trusted workflows. `ci.yml` has no `pull_request` trigger on purpose: a fork PR can edit the workflow and run on this machine. -Jobs use `runs-on: [self-hosted, linux]`, which matches the default labels GitHub gives a Linux runner. -A self-hosted runner has no tool image, so everything the jobs call must be installed on the machine. +The runner is the `myoung34/github-runner:debian-bookworm` image. Jobs use `runs-on: [self-hosted, linux]`, which matches the image's default `self-hosted`, `Linux` and `X64`/`ARM64` labels. +The image has no ruff, uv, gitleaks or actionlint, so each job installs what it needs with `.github/scripts/setup-tools.sh`. +The script downloads pinned releases, verifies their SHA-256 checksums, caches them under `$RUNNER_TOOL_CACHE`, and adds them to `PATH`. +To bump a tool, change the version at the top of the script. +The script needs `curl`, `tar`, `sha256sum` and `install`, which the image provides. -1. Repo → Settings → Actions → Runners → **New self-hosted runner** → Linux (pick x64 or ARM64 to match `uname -m`). - Follow the download and `./config.sh --url ... --token ...` steps in `~/actions-runner`. - Keep the default labels (`self-hosted`, `Linux`, plus `X64` or `ARM64`). - Run it as a normal user, not root. -2. Install as a systemd service: `sudo ./svc.sh install && sudo ./svc.sh start` - (`sudo ./svc.sh install ` to run as a specific user). -3. Install the base packages: `sudo apt-get update && sudo apt-get install -y git curl ca-certificates jq tar` -4. Install the tools the workflow calls directly. `uv` provides `uvx ruff` and `uvx pip-audit`, and downloads the Python 3.11 that the hygiene job needs: +## docker-compose - ```bash - curl -LsSf https://astral.sh/uv/install.sh | sh # installs to ~/.local/bin +```yaml +services: + github-runner: + image: myoung34/github-runner:debian-bookworm + restart: always + environment: + REPO_URL: "https://github.com/nprasad2077/MNIST" + RUNNER_TOKEN: "" # Settings → Actions → Runners → New self-hosted runner + RUNNER_NAME: "coolify-debian-runner" + RUNNER_WORKDIR: "/tmp/runner/work" + # No docker.sock mount: the jobs don't use Docker, and the socket gives every job root on the host. +``` - mkdir -p ~/.local/bin && cd "$(mktemp -d)" - case "$(uname -m)" in x86_64) A=x64 ;; aarch64) A=arm64 ;; esac +- The registration token expires after about an hour. Keep it out of git; set it in Coolify's environment variables, not in this file. +- The container filesystem is disposable. When the container is recreated, the next run downloads the tools again. - curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_${A}.tar.gz" \ - | tar xz gitleaks && install gitleaks ~/.local/bin/ +## Repo setting - curl -sSfL "https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_${A/x64/amd64}.tar.gz" \ - | tar xz actionlint && install actionlint ~/.local/bin/ - ``` +Settings → Actions → General → Fork pull request workflows → **Require approval for all outside collaborators**. - Optional: `sudo apt-get install -y shellcheck`. `actionlint` runs it on `run:` scripts if present. -5. The service does not load shell profiles. Make sure `~/.local/bin` is on the service's PATH by adding this line to - `~/actions-runner/.env`, then `sudo ./svc.sh stop && sudo ./svc.sh start`: - - ``` - PATH=/home//.local/bin:/usr/local/bin:/usr/bin:/bin - ``` - -6. Repo → Settings → Actions → General → Fork pull request workflows → - **Require approval for all outside collaborators**. - -Check the setup with `sudo -u env -i PATH=/home//.local/bin:/usr/bin:/bin sh -c 'uvx --version && gitleaks version && actionlint -version && jq --version'`. +Check the container with `docker exec bash -c 'cd "$(mktemp -d)" && curl -sSfL https://raw.githubusercontent.com/nprasad2077/MNIST/dev/.github/scripts/setup-tools.sh | bash -s uv gitleaks actionlint'`. diff --git a/.github/scripts/setup-tools.sh b/.github/scripts/setup-tools.sh new file mode 100644 index 0000000..4bef1fe --- /dev/null +++ b/.github/scripts/setup-tools.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Install pinned CI tools (Linux x64/arm64) on the self-hosted runner and put them on PATH. +# Usage: setup-tools.sh [uv] [gitleaks] [actionlint] +# Needs only curl, tar and sha256sum. Downloads are verified against the published checksums. +set -euo pipefail + +UV_VERSION=0.10.7 +GITLEAKS_VERSION=8.30.1 +ACTIONLINT_VERSION=1.7.12 + +# Versions are part of the path, so bumping a pin installs into a fresh directory. +BIN="${RUNNER_TOOL_CACHE:-$HOME/.cache}/mnist-ci/uv${UV_VERSION}-gl${GITLEAKS_VERSION}-al${ACTIONLINT_VERSION}/bin" + +case "$(uname -m)" in + x86_64) UV_ARCH=x86_64 GL_ARCH=x64 AL_ARCH=amd64 ;; + aarch64 | arm64) UV_ARCH=aarch64 GL_ARCH=arm64 AL_ARCH=arm64 ;; + *) echo "::error::unsupported architecture $(uname -m)"; exit 1 ;; +esac + +# Download /, verify it against /, extract into $BIN. +install_tarball() { + local base=$1 archive=$2 sums=$3 strip=$4 + shift 4 + local tmp + tmp=$(mktemp -d) + curl -sSfL --retry 3 -o "$tmp/$archive" "$base/$archive" + curl -sSfL --retry 3 -o "$tmp/$sums" "$base/$sums" + (cd "$tmp" && grep -F " $archive" "$sums" | sha256sum -c -) + tar -xzf "$tmp/$archive" -C "$tmp" + mkdir -p "$BIN" + for member in "$@"; do + install -m 0755 "$tmp/${strip}${member}" "$BIN/$member" + done + rm -rf "$tmp" +} + +for tool in "$@"; do + if [ -x "$BIN/$tool" ]; then + continue + fi + case "$tool" in + uv) + triple="uv-${UV_ARCH}-unknown-linux-gnu" + # uv publishes one .sha256 per archive; reuse the same verification path. + base="https://github.com/astral-sh/uv/releases/download/${UV_VERSION}" + tmp=$(mktemp -d) + curl -sSfL --retry 3 -o "$tmp/$triple.tar.gz" "$base/$triple.tar.gz" + curl -sSfL --retry 3 -o "$tmp/$triple.tar.gz.sha256" "$base/$triple.tar.gz.sha256" + (cd "$tmp" && sha256sum -c "$triple.tar.gz.sha256") + tar -xzf "$tmp/$triple.tar.gz" -C "$tmp" + mkdir -p "$BIN" + install -m 0755 "$tmp/$triple/uv" "$tmp/$triple/uvx" "$BIN/" + rm -rf "$tmp" + ;; + gitleaks) + install_tarball "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" \ + "gitleaks_${GITLEAKS_VERSION}_linux_${GL_ARCH}.tar.gz" "gitleaks_${GITLEAKS_VERSION}_checksums.txt" "" gitleaks + ;; + actionlint) + install_tarball "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}" \ + "actionlint_${ACTIONLINT_VERSION}_linux_${AL_ARCH}.tar.gz" "actionlint_${ACTIONLINT_VERSION}_checksums.txt" "" actionlint + ;; + *) echo "::error::unknown tool: $tool"; exit 1 ;; + esac +done + +if [ -n "${GITHUB_PATH:-}" ]; then + echo "$BIN" >> "$GITHUB_PATH" +fi +export PATH="$BIN:$PATH" +for tool in "$@"; do + case "$tool" in + gitleaks) "$tool" version ;; + actionlint) "$tool" -version | head -1 ;; + *) "$tool" --version ;; + esac +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f21045c..706bf05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - run: bash .github/scripts/setup-tools.sh uv - run: uvx "ruff@${RUFF_VERSION}" check --output-format=github notebooks/ format-notebooks: @@ -33,6 +34,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - run: bash .github/scripts/setup-tools.sh uv - run: uvx "ruff@${RUFF_VERSION}" format --check notebooks/ secret-scan: @@ -42,6 +44,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - run: bash .github/scripts/setup-tools.sh gitleaks - run: gitleaks detect --source . --redact --verbose dep-audit: @@ -51,6 +54,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - run: bash .github/scripts/setup-tools.sh uv # Audit the pins only; installing requirements.txt fails on pywinpty. - run: uvx pip-audit -r requirements.txt --no-deps --disable-pip @@ -60,6 +64,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - run: bash .github/scripts/setup-tools.sh actionlint - run: actionlint notebook-hygiene: @@ -68,6 +73,7 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - run: bash .github/scripts/setup-tools.sh uv - run: uv run --python 3.11 --no-project python .github/scripts/notebook_hygiene.py summary: @@ -77,7 +83,12 @@ jobs: steps: - name: Write job summary env: - RESULTS: ${{ toJSON(needs) }} + LINT: ${{ needs.lint-notebooks.result }} + FORMAT: ${{ needs.format-notebooks.result }} + SECRETS: ${{ needs.secret-scan.result }} + AUDIT: ${{ needs.dep-audit.result }} + WORKFLOW: ${{ needs.workflow-lint.result }} + HYGIENE: ${{ needs.notebook-hygiene.result }} EVENT: ${{ github.event_name }} REF_NAME: ${{ github.ref_name }} ACTOR: ${{ github.actor }} @@ -88,7 +99,12 @@ jobs: echo echo "| Job | Result |" echo "|---|---|" - echo "$RESULTS" | jq -r 'to_entries[] | "| \(.key) | \(.value.result) |"' + echo "| lint-notebooks | $LINT |" + echo "| format-notebooks | $FORMAT |" + echo "| secret-scan | $SECRETS |" + echo "| dep-audit | $AUDIT |" + echo "| workflow-lint | $WORKFLOW |" + echo "| notebook-hygiene | $HYGIENE |" echo echo "- **Trigger:** $EVENT" echo "- **Ref:** $REF_NAME @ ${GITHUB_SHA::7}"