diff --git a/.github/RUNNER_SETUP.md b/.github/RUNNER_SETUP.md new file mode 100644 index 0000000..309a48c --- /dev/null +++ b/.github/RUNNER_SETUP.md @@ -0,0 +1,19 @@ +# Self-hosted runner setup (macOS arm64) + +This repo is public, so only run the runner for trusted workflows. `ci.yml` skips fork PRs. + +1. Repo → Settings → Actions → Runners → **New self-hosted runner** → macOS / ARM64. + Follow the download and `./config.sh --url ... --token ...` steps in `~/actions-runner`. + Keep the default labels (`self-hosted`, `macOS`, `ARM64`). +2. Install as a launchd service: `./svc.sh install && ./svc.sh start`. +3. The service does not load shell profiles. Add the tool directories to `~/actions-runner/.path` + (one `PATH` line, then `./svc.sh stop && ./svc.sh start`): + + ``` + /Users//.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + ``` + +4. Install the tools the workflow calls directly: `brew install gitleaks actionlint jq` + (`uv` provides `uvx ruff` and `uvx pip-audit`). +5. Repo → Settings → Actions → General → Fork pull request workflows → + **Require approval for all outside collaborators**. diff --git a/.github/scripts/notebook_hygiene.py b/.github/scripts/notebook_hygiene.py new file mode 100644 index 0000000..ada9da4 --- /dev/null +++ b/.github/scripts/notebook_hygiene.py @@ -0,0 +1,48 @@ +"""Check that notebooks are valid JSON and report their size and embedded outputs. + +Exits 1 only for unreadable notebooks; oversized files are warnings. +""" + +import json +import sys +from pathlib import Path + +SIZE_WARN_BYTES = 500 * 1024 + + +def count_png_outputs(nb: dict) -> int: + return sum( + 1 + for cell in nb.get("cells", []) + for out in cell.get("outputs", []) + if "image/png" in out.get("data", {}) + ) + + +def main(root: str = "notebooks") -> int: + paths = sorted(Path(root).glob("*.ipynb")) + if not paths: + print(f"::error::no notebooks found under {root}/") + return 1 + + failed = False + print(f"{'notebook':<28}{'KB':>8}{'cells':>7}{'png':>5}") + for path in paths: + size = path.stat().st_size + try: + nb = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"::error file={path}::invalid notebook JSON: {exc}") + failed = True + continue + print( + f"{path.name:<28}{size / 1024:>8.0f}" + f"{len(nb.get('cells', [])):>7}{count_png_outputs(nb):>5}" + ) + if size > SIZE_WARN_BYTES: + print(f"::warning file={path}::{size / 1024:.0f} KB exceeds {SIZE_WARN_BYTES // 1024} KB") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(*sys.argv[1:2])) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4d8926f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,104 @@ +name: CI + +on: + push: + branches: [main, dev, actions] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +# Public repo + self-hosted runner: never run fork PRs on this machine. +env: + RUFF_VERSION: "0.15.0" + +jobs: + lint-notebooks: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - run: uvx "ruff@${RUFF_VERSION}" check --output-format=github notebooks/ + + format-notebooks: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + continue-on-error: true # advisory: notebooks were never formatted + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - run: uvx "ruff@${RUFF_VERSION}" format --check notebooks/ + + secret-scan: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - run: gitleaks detect --source . --redact --verbose + + dep-audit: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + continue-on-error: true # advisory: pins are stale; Dependabot tracks bumps + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + # Audit the pins only; installing requirements.txt fails on pywinpty. + - run: uvx pip-audit -r requirements.txt --no-deps --disable-pip + + workflow-lint: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - run: actionlint + + notebook-hygiene: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - run: uv run --python 3.11 --no-project python .github/scripts/notebook_hygiene.py + + summary: + if: always() + needs: [lint-notebooks, format-notebooks, secret-scan, dep-audit, workflow-lint, notebook-hygiene] + runs-on: self-hosted + steps: + - name: Write job summary + env: + RESULTS: ${{ toJSON(needs) }} + EVENT: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + ACTOR: ${{ github.actor }} + RUNNER_DESC: ${{ runner.name }} (${{ runner.os }}/${{ runner.arch }}) + run: | + { + echo "## CI results" + echo + echo "| Job | Result |" + echo "|---|---|" + echo "$RESULTS" | jq -r 'to_entries[] | "| \(.key) | \(.value.result) |"' + echo + echo "- **Trigger:** $EVENT" + echo "- **Ref:** $REF_NAME @ ${GITHUB_SHA::7}" + echo "- **Actor:** $ACTOR" + echo "- **Runner:** $RUNNER_DESC" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/notebooks/04_advanced_cnn.ipynb b/notebooks/04_advanced_cnn.ipynb index f70b655..3baa0fb 100644 --- a/notebooks/04_advanced_cnn.ipynb +++ b/notebooks/04_advanced_cnn.ipynb @@ -22,7 +22,7 @@ "import duckdb\n", "import torch\n", "import torch.nn as nn\n", - "from torch.utils.data import DataLoader, TensorDataset, Dataset\n", + "from torch.utils.data import DataLoader, Dataset\n", "import torchvision.transforms as T\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..0ef5248 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,7 @@ +target-version = "py311" +extend-exclude = ["docs"] + +[lint] +select = ["E", "F", "W"] +# Notebooks import where they first need a package, run long lines, and chain matplotlib calls with ";". +ignore = ["E402", "E501", "E702"]