From 82f1f974f149f4795764155319d90ae26b461cfc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:27:35 +0100 Subject: [PATCH 1/8] feat(scripts): add a script to delete unused manifests - v3 and 4.0.0-beta.420.x do not follow semantic versioning, KEEP_TAG_EXCEPTIONS keeps them explicitly --- .editorconfig | 6 + scripts/clean-manifests.sh | 267 +++++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100755 scripts/clean-manifests.sh diff --git a/.editorconfig b/.editorconfig index 8f0de65c56..c998050b7f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,3 +16,9 @@ indent_size = 2 [docker-compose.yml] indent_size = 4 + +[scripts/clean-manifests.sh] +indent_style = tab +binary_next_line = true +switch_case_indent = true +space_redirects = true diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh new file mode 100755 index 0000000000..b2a36aee1c --- /dev/null +++ b/scripts/clean-manifests.sh @@ -0,0 +1,267 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +LIMIT="${2:-}" +IMAGE=coollabsio/coolify +KEEP_TAG_PATTERN='^(latest|preview|main|nightly|v?[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' +KEEP_TAG_EXCEPTIONS='v3 4.0.0-beta.420.1 4.0.0-beta.420.2 4.0.0-beta.420.3 4.0.0-beta.420.4 4.0.0-beta.420.5 4.0.0-beta.420.6 4.0.0-beta.420.7 4.0.0-beta.420.8 4.0.0-beta.420.9 4.0.0-beta.420.10' +KEEP_NIGHTLY_DAYS=14 +DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" +DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" +TO_DELETE_FILE=scripts/manifests-to-delete.tsv +DELETED_FILE=scripts/deleted-manifests.tsv +WORK_DIR="$(mktemp -d)" + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." +trap 'rm -rf "$WORK_DIR"' EXIT + +main() { + [[ "$OPTION" =~ ^(--delete [1-9][0-9]*)?$ ]] || die "'$OPTION' is not an option, expected --delete with a count or none" + heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" + read_ghcr_versions + read_dockerhub_tags + select_kept_tags + read_manifests + select_kept_manifests + heading "Selecting what to delete" + find_dockerhub_only_manifests + write_list + delete_manifests + verify_kept_manifests +} + +read_ghcr_versions() { + local state + for state in active deleted; do + gh api --paginate "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&state=$state" \ + --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' > "$WORK_DIR/ghcr-$state.tsv" 2> /dev/null \ + || die "gh could not list the $state versions of ghcr.io/$IMAGE, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" + done + cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and lists $(count < "$WORK_DIR/ghcr-deleted.tsv") deleted ones" +} + +read_dockerhub_tags() { + local token page last="" + token="$(registry_token docker.io pull)" + while :; do + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/tags/list?n=1000${last:+&last=$last}" | jq -r '.tags // [] | .[]')" \ + || die "could not list the tags of docker.io/$IMAGE" + [ -n "$page" ] || break + echo "$page" >> "$WORK_DIR/dockerhub-tags.txt" + last="$(tail -n 1 <<< "$page")" + done + request_manifests docker.io "$WORK_DIR/dockerhub-tags.txt" -f --head -o /dev/null -w '%header{docker-content-digest}\t%header{content-type}\t{}\n' \ + > "$WORK_DIR/dockerhub.tsv" || die "could not read the digests of the tags of docker.io/$IMAGE" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(cut -f 1 "$WORK_DIR/dockerhub.tsv" | sort -u | count) manifests" +} + +select_kept_tags() { + local cutoff + { cut -f 3 "$WORK_DIR/ghcr-active.tsv" | tr ',' '\n' && cut -f 3 "$WORK_DIR/dockerhub.tsv"; } | sed '/^$/d' | sort -u > "$WORK_DIR/tags.txt" + git ls-remote --tags "https://github.com/$IMAGE.git" | sed -n 's|.*refs/tags/||p' | sed 's/\^{}$//; p; s/^v//' > "$WORK_DIR/keep.txt" \ + || die "could not list the git tags of github.com/$IMAGE" + gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' >> "$WORK_DIR/keep.txt" \ + || die "gh could not list the open pull requests of $IMAGE" + tr ' ' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" + cutoff="$(jq -nr --argjson days "$KEEP_NIGHTLY_DAYS" 'now - $days * 86400 | strftime("%Y%m%d")')" + { + grep -E "$KEEP_TAG_PATTERN" "$WORK_DIR/tags.txt" || true + grep -xF -f "$WORK_DIR/keep.txt" "$WORK_DIR/tags.txt" || true + grep -E -e '-(alpha|beta|rc)\.[0-9]+\.20[0-9]{6}\.[0-9]+$' "$WORK_DIR/tags.txt" | awk -F . -v cutoff="$cutoff" '$(NF - 1) >= cutoff' || true + } | sort -u > "$WORK_DIR/kept-tags.txt" + grep -qx latest "$WORK_DIR/kept-tags.txt" || die "found no latest tag, expected it among the tags to keep" + ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: every tag KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, the nightlies of the last $KEEP_NIGHTLY_DAYS days, every git tag of github.com/$IMAGE and the tags of open pull requests" +} + +read_manifests() { + local digest tag + mkdir "$WORK_DIR/manifests" + request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ + || die "could not read the manifests of ghcr.io/$IMAGE" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" + awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the indexes kept tags point at on docker.io" + while IFS=$'\t' read -r digest tag; do + curl -sSf --retry 3 "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ + || die "could not read the manifests $tag points at on docker.io" + done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ + >> "$WORK_DIR/children.tsv" + manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" + manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ + > "$WORK_DIR/kinds.tsv" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests of ghcr.io/$IMAGE and the $(count < "$WORK_DIR/dockerhub-roots.tsv") that kept tags point at on docker.io alone" +} + +select_kept_manifests() { + local kinds + kept_manifests "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" + kept_manifests "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" + sort -u "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kept-docker.io.txt" > "$WORK_DIR/kept.txt" + awk 'FILENAME == ARGV[1] { ghcr[$1]; next } !($1 in ghcr) { exit 1 }' "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/kept-ghcr.io.txt" \ + || die "manifests the kept tags point at are missing from the versions gh listed on ghcr.io, expected every one of them" + kinds="$(awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } + END { printf "%d indexes, %d images and %d attestations", count["index"], count["image"], count["attestation"] }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" + ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io, $kinds, and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io: the kept tags and every manifest they point at" +} + +find_dockerhub_only_manifests() { + awk -F '\t' 'FILENAME == ARGV[1] || FILENAME == ARGV[2] { skip[$1]; next } !($1 in skip) && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/candidates.txt" + request_manifests docker.io "$WORK_DIR/candidates.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' \ + > "$WORK_DIR/dockerhub-only.tsv" || die "could not check which manifests docker.io has" + awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/dockerhub-only.tsv" \ + || die "could not check docker.io for the manifests ghcr.io does not have, expected 200 or 404" + ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/dockerhub-only.tsv" | count) of the $(count < "$WORK_DIR/candidates.txt") unused manifests ghcr.io does not have" +} + +write_list() { + awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + FILENAME == ARGV[1] { kept[$1]; next } + FILENAME == ARGV[2] { kind[$1] = $2; next } + FILENAME == ARGV[3] { size[$1] = $2; next } + FILENAME == ARGV[4] { size[$1] += $3; next } + FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } + FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } + FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } + FILENAME == ARGV[8] { if ($1 == 200) print "docker.io", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } + !($1 in kept) { print "ghcr.io and docker.io", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k2,2r -k5,5 > "$WORK_DIR/rows.tsv" + write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" + ok "$(awk -F '\t' 'END { print $8 " to delete, " $2 ", they free " $6 }' "$TO_DELETE_FILE")" + ok "$(awk -F '\t' '$1 == "docker.io"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" + info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" +} + +delete_manifests() { + local registries type tags version digest row name deleted status + if [ -z "$LIMIT" ]; then + info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" + return + fi + [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" + heading "Deleting" + write_table /dev/null "$DELETED_FILE" + while IFS='|' read -r registries type tags version digest row; do + name="$digest${tags:+ ($tags)}" + deleted="" + if [ "$registries" != docker.io ]; then + delete_ghcr_version "$version" "$type $name" || continue + deleted=ghcr.io + fi + status="$(delete_dockerhub_manifest "$digest")" + [[ "$status" != 2?? ]] || deleted="${deleted:+$deleted and }docker.io" + if [ -n "$deleted" ]; then + ok "deleted the $type $BOLD$name$RESET from $deleted" + printf '%s\t%s\n' "$deleted" "${row#*$'\t'}" >> "$WORK_DIR/deleted.tsv" + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + fi + case "$status" in + 2?? | 404) ;; + 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; + *) die "could not delete the $type $name from docker.io, it answered $status" ;; + esac + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ print $1, $2, $4, $7, $8, $0 }') + ok "$(awk -F '\t' 'END { print "deleted " $8 ", " $2 ", they free " $6 }' "$DELETED_FILE")" + info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" +} + +verify_kept_manifests() { + local registry missing + [ -n "$LIMIT" ] || return 0 + for registry in ghcr.io docker.io; do + missing="$(request_manifests "$registry" "$WORK_DIR/kept-$registry.txt" --head -o /dev/null -w '%{http_code}\t{}\n' | awk '$1 != 200 { print $2 }')" \ + || die "could not check the kept manifests on $registry" + [ -z "$missing" ] || die "$(count <<< "$missing") manifests the kept tags point at are gone from $registry, expected all of them: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" + done + ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" +} + +request_manifests() { + local registry="$1" list="$2" chunk token + shift 2 + [ -s "$list" ] || return 0 + split -l 1000 "$list" "$WORK_DIR/chunk-" + for chunk in "$WORK_DIR"/chunk-*; do + token="$(registry_token "$registry" pull)" + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + rm "$chunk" + done +} + +manifest_fields() { + find "$WORK_DIR/manifests" -type f -exec jq -r 'def manifest: input_filename | sub(".*/"; ""); '"$1"' | @tsv' {} + +} + +kept_manifests() { + awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } + FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[2] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" +} + +write_table() { + awk -F '\t' -v OFS='\t' 'function human(bytes, unit) { for (unit = 1; bytes >= 1000 && unit < 5; unit++) bytes /= 1000; return sprintf(unit == 1 ? "%d %s" : "%.1f %s", bytes, units[unit]) } + function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } + BEGIN { split("B kB MB GB TB", units, " "); print "registries", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } + FILENAME == ARGV[1] { removed[$8]; types[$2]++; manifests++; if ($6 != "") $6 = human($6); print; next } + FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } + { blob[$2] = $3; if (!($1 in removed)) used[$2] } + END { + for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } + print "total", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), + "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") + }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" +} + +delete_ghcr_version() { + local error + until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do + case "$error" in + *"rate limit"*) sleep 60 ;; + *downloads*) + warn "ghcr.io and docker.io keep the $2, it has more than 5,000 downloads on ghcr.io" + return 1 + ;; + *) die "could not delete the $2 from ghcr.io: $error" ;; + esac + done + sleep 1 +} + +delete_dockerhub_manifest() { + local status + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ + "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + sleep 60 + done + echo "$status" +} + +registry_token() { + case "$1" in + ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; + docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + esac | jq -r .token +} + +count() { awk 'END { print NR }'; } + +main From 26f8c990a540335624120862e07d1af35fdd9887 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:27:40 +0100 Subject: [PATCH 2/8] ci: delete unused manifests every hour - with a run every hour and 200 deletions per run the about 35,000 unused manifests on ghcr.io and docker.io are deleted in about 7 days, then the schedule goes back to every 2 weeks or every month - a run deletes 200 at most, as GitHub allows an account 500 requests per hour and the ghrc.io PAT is an account level PAT --- .github/workflows/clean-manifests.yaml | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/clean-manifests.yaml diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml new file mode 100644 index 0000000000..bd1899214f --- /dev/null +++ b/.github/workflows/clean-manifests.yaml @@ -0,0 +1,51 @@ +name: Clean manifests + +on: + schedule: + - cron: "20 * * * *" + workflow_dispatch: + +permissions: {} + +cache-mode: none + +concurrency: + group: clean-manifests + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + clean: + name: Clean + if: ${{ github.repository == 'coollabsio/coolify' }} + permissions: + contents: read + environment: + name: cleanup + deployment: false + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 60 + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Delete the unused manifests + env: + GH_TOKEN: ${{ secrets.GHCR_CLEANUP_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CLEANUP_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} + run: scripts/clean-manifests.sh --delete 200 + + - name: Upload the list of deleted manifests + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: deleted-manifests + path: scripts/deleted-manifests.tsv + if-no-files-found: ignore + retention-days: 90 From c5d380a4c61fe1b9535c37c3e16f9faaf3d884fc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:27:40 +0100 Subject: [PATCH 3/8] feat(scripts): give each registry a column in the lists - docker_hub is undefined for a manifest ghcr.io has, as a HEAD on docker.io for every one of them could trigger rate limits --- scripts/clean-manifests.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index b2a36aee1c..98938c023f 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -138,18 +138,18 @@ write_list() { FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } - FILENAME == ARGV[8] { if ($1 == 200) print "docker.io", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } - !($1 in kept) { print "ghcr.io and docker.io", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } + !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k2,2r -k5,5 > "$WORK_DIR/rows.tsv" + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" - ok "$(awk -F '\t' 'END { print $8 " to delete, " $2 ", they free " $6 }' "$TO_DELETE_FILE")" - ok "$(awk -F '\t' '$1 == "docker.io"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" + ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" + ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" } delete_manifests() { - local registries type tags version digest row name deleted status + local on_ghcr type tags version digest rest name deleted ghcr docker_hub status if [ -z "$LIMIT" ]; then info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" return @@ -157,18 +157,20 @@ delete_manifests() { [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" heading "Deleting" write_table /dev/null "$DELETED_FILE" - while IFS='|' read -r registries type tags version digest row; do + while IFS='|' read -r on_ghcr type tags version digest rest; do name="$digest${tags:+ ($tags)}" - deleted="" - if [ "$registries" != docker.io ]; then + deleted="" ghcr=no docker_hub=no + if [ "$on_ghcr" = yes ]; then delete_ghcr_version "$version" "$type $name" || continue - deleted=ghcr.io + deleted=ghcr.io ghcr=yes fi status="$(delete_dockerhub_manifest "$digest")" - [[ "$status" != 2?? ]] || deleted="${deleted:+$deleted and }docker.io" + if [[ "$status" == 2?? ]]; then + deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + fi if [ -n "$deleted" ]; then ok "deleted the $type $BOLD$name$RESET from $deleted" - printf '%s\t%s\n' "$deleted" "${row#*$'\t'}" >> "$WORK_DIR/deleted.tsv" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" fi case "$status" in @@ -176,8 +178,8 @@ delete_manifests() { 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac - done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ print $1, $2, $4, $7, $8, $0 }') - ok "$(awk -F '\t' 'END { print "deleted " $8 ", " $2 ", they free " $6 }' "$DELETED_FILE")" + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') + ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" } @@ -220,13 +222,13 @@ kept_manifests() { write_table() { awk -F '\t' -v OFS='\t' 'function human(bytes, unit) { for (unit = 1; bytes >= 1000 && unit < 5; unit++) bytes /= 1000; return sprintf(unit == 1 ? "%d %s" : "%.1f %s", bytes, units[unit]) } function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } - BEGIN { split("B kB MB GB TB", units, " "); print "registries", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } - FILENAME == ARGV[1] { removed[$8]; types[$2]++; manifests++; if ($6 != "") $6 = human($6); print; next } + BEGIN { split("B kB MB GB TB", units, " "); print "ghcr", "docker_hub", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } + FILENAME == ARGV[1] { removed[$9]; types[$3]++; manifests++; if ($7 != "") $7 = human($7); print; next } FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } { blob[$2] = $3; if (!($1 in removed)) used[$2] } END { for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } - print "total", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), + print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" } From 200042e16bc50f7a207b23870e9abfd4b89bdfb3 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:27:40 +0100 Subject: [PATCH 4/8] feat(scripts): check twice the delete limit on docker.io - ghcr.io lists deleted versions for 30 days, a HEAD on docker.io for each would reach thousands per run, which could trigger rate limits - twice the delete limit covers the last two runs, and manifests only docker.io has sort first, so a copy docker.io refused is deleted within two runs --- scripts/clean-manifests.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 98938c023f..43ef004eea 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -43,14 +43,13 @@ main() { } read_ghcr_versions() { - local state - for state in active deleted; do - gh api --paginate "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&state=$state" \ - --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' > "$WORK_DIR/ghcr-$state.tsv" 2> /dev/null \ - || die "gh could not list the $state versions of ghcr.io/$IMAGE, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" - done + local page window=$((${LIMIT:-0} * 2)) + list_ghcr_versions "state=active" --paginate > "$WORK_DIR/ghcr-active.tsv" + for ((page = 1; page <= (window + 99) / 100; page++)); do + list_ghcr_versions "state=deleted&page=$page" + done | awk -v window="$window" 'NR <= window' > "$WORK_DIR/ghcr-deleted.tsv" cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" - ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and lists $(count < "$WORK_DIR/ghcr-deleted.tsv") deleted ones" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and $(count < "$WORK_DIR/ghcr-deleted.tsv") of the versions it deleted last are checked on docker.io, twice as many as the run deletes" } read_dockerhub_tags() { @@ -141,7 +140,7 @@ write_list() { FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/rows.tsv" + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" @@ -194,6 +193,14 @@ verify_kept_manifests() { ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" } +list_ghcr_versions() { + local query="$1" + shift + gh api "$@" "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&$query" \ + --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' 2> /dev/null \ + || die "gh could not list the versions of ghcr.io/$IMAGE with $query, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" +} + request_manifests() { local registry="$1" list="$2" chunk token shift 2 From 4b7b9bc14505d2ec00d7369565166027676f666b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:35:49 +0100 Subject: [PATCH 5/8] ci: run the workflow as a dry run first - a manually triggered first run in CI uploads the full list of manifests to delete - a later pull request makes the workflow hourly again --- .github/workflows/clean-manifests.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index bd1899214f..28cfebfebb 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -1,8 +1,6 @@ name: Clean manifests on: - schedule: - - cron: "20 * * * *" workflow_dispatch: permissions: {} @@ -34,18 +32,18 @@ jobs: with: persist-credentials: false - - name: Delete the unused manifests + - name: List the manifests to delete env: GH_TOKEN: ${{ secrets.GHCR_CLEANUP_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CLEANUP_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} - run: scripts/clean-manifests.sh --delete 200 + run: scripts/clean-manifests.sh - - name: Upload the list of deleted manifests + - name: Upload the list of manifests to delete if: ${{ always() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: deleted-manifests - path: scripts/deleted-manifests.tsv + name: manifests-to-delete + path: scripts/manifests-to-delete.tsv if-no-files-found: ignore retention-days: 90 From 0375750aed3fc28f883dbb028e6a850ab4093245 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:39:46 +0100 Subject: [PATCH 6/8] feat(scripts): show the ghcr.io sizes in brackets --- scripts/clean-manifests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 43ef004eea..95e4a081e0 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -142,7 +142,7 @@ write_list() { "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" - ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? $9 " to delete, " $3 ", they free " $7 : "nothing to delete") }' "$TO_DELETE_FILE")" ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" } @@ -178,7 +178,7 @@ delete_manifests() { *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') - ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? "deleted " $9 ", " $3 ", they free " $7 : "deleted nothing") }' "$DELETED_FILE")" info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" } @@ -236,7 +236,7 @@ write_table() { END { for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), - "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") + "", "", "", human(before - after) " on ghcr.io (" human(before) " → " human(after) ")", "", amount(manifests, "manifest", "manifests") }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" } From b001e80fc91230c2b6f358f66c28c85f2c568308 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:15:55 +0100 Subject: [PATCH 7/8] feat(scripts): delete only versions older than a day - a build pushes its images before the index that tags them, a run in between would see them as unused - a push while the run reads the pages of versions shows one version twice, the list keeps it once so the second delete does not stop the run --- scripts/clean-manifests.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 95e4a081e0..7b1fa25799 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -7,6 +7,7 @@ IMAGE=coollabsio/coolify KEEP_TAG_PATTERN='^(latest|preview|main|nightly|v?[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' KEEP_TAG_EXCEPTIONS='v3 4.0.0-beta.420.1 4.0.0-beta.420.2 4.0.0-beta.420.3 4.0.0-beta.420.4 4.0.0-beta.420.5 4.0.0-beta.420.6 4.0.0-beta.420.7 4.0.0-beta.420.8 4.0.0-beta.420.9 4.0.0-beta.420.10' KEEP_NIGHTLY_DAYS=14 +MIN_AGE_DAYS=1 DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" TO_DELETE_FILE=scripts/manifests-to-delete.tsv @@ -129,7 +130,10 @@ find_dockerhub_only_manifests() { } write_list() { - awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + local cutoff + cutoff="$(jq -nr --argjson days "$MIN_AGE_DAYS" 'now - $days * 86400 | strftime("%Y-%m-%dT%H:%M:%SZ")')" + ok "keeping the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff && !seen[$1]++' "$WORK_DIR/ghcr-active.tsv" | count) versions ghcr.io got since $cutoff, a build may still push the index that needs them" + awk -F '\t' -v OFS='\t' -v cutoff="$cutoff" 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { kind[$1] = $2; next } FILENAME == ARGV[3] { size[$1] = $2; next } @@ -138,7 +142,7 @@ write_list() { FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } - !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + !($1 in kept) && $4 < cutoff && !seen[$1]++ { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" From fb0210f582cd519f88cc8d28074758074f84d57e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:56:31 +0100 Subject: [PATCH 8/8] feat(scripts): harden the requests of the manifest cleanup - passes the Docker Hub login to curl on stdin, which keeps it out of the process list - checks the format of version ids and digests before a delete --- scripts/clean-manifests.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 7b1fa25799..228b550b62 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -161,6 +161,7 @@ delete_manifests() { heading "Deleting" write_table /dev/null "$DELETED_FILE" while IFS='|' read -r on_ghcr type tags version digest rest; do + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "the list names '$digest', expected a sha256 digest" name="$digest${tags:+ ($tags)}" deleted="" ghcr=no docker_hub=no if [ "$on_ghcr" = yes ]; then @@ -246,6 +247,7 @@ write_table() { delete_ghcr_version() { local error + [[ "$1" =~ ^[0-9]+$ ]] || die "the version id of the $2 is '$1', expected a number from the GitHub API" until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do case "$error" in *"rate limit"*) sleep 60 ;; @@ -271,10 +273,14 @@ delete_dockerhub_manifest() { registry_token() { case "$1" in ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; - docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + docker.io) basic_auth "${DOCKERHUB_USERNAME:+$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN}" | curl -sSf --retry 3 -K - "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; esac | jq -r .token } +basic_auth() { + [ -z "$1" ] || printf 'header = "Authorization: Basic %s"\n' "$(printf '%s' "$1" | jq -Rrj @base64)" +} + count() { awk 'END { print NR }'; } main