mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-04 10:42:23 +00:00
449 lines
20 KiB
PHP
449 lines
20 KiB
PHP
<?php
|
|
|
|
namespace App\Actions\Server;
|
|
|
|
use App\Enums\ApplicationDeploymentStatus;
|
|
use App\Models\ApplicationDeploymentQueue;
|
|
use App\Models\Server;
|
|
use App\Support\Actions\UniqueUntilProcessingJobDecorator;
|
|
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Lorisleiva\Actions\Concerns\AsAction;
|
|
use Lorisleiva\Actions\Decorators\JobDecorator;
|
|
use Lorisleiva\Actions\Decorators\UniqueJobDecorator;
|
|
use RuntimeException;
|
|
|
|
class CleanupDocker implements ShouldBeUnique
|
|
{
|
|
use AsAction;
|
|
|
|
/**
|
|
* Timeout of a queued cleanup, shared with DockerCleanupJob.
|
|
*/
|
|
public const JOB_TIMEOUT = 600;
|
|
|
|
/**
|
|
* Time budget for all remote commands of one cleanup. It stays below JOB_TIMEOUT, so a
|
|
* slow server makes the cleanup fail with an error instead of the worker killing the job.
|
|
* A remote Docker prune that already started still finishes on the server.
|
|
*/
|
|
public const REMOTE_COMMANDS_DEADLINE = 540;
|
|
|
|
/**
|
|
* Delay of a queued cleanup. Stop requests for one server in this window (for example a
|
|
* bulk stop through the API) become one cleanup that runs after the batch.
|
|
*/
|
|
public const QUEUED_DELAY = 60;
|
|
|
|
/**
|
|
* A resource stop does not queue a cleanup when a cleanup finished on the server in this
|
|
* window. The scheduled cleanup still runs.
|
|
*/
|
|
public const STOP_CLEANUP_COOLDOWN = 3600;
|
|
|
|
/**
|
|
* Cleanup log command of the entry that records a skipped build cache prune.
|
|
*/
|
|
public const BUILD_CACHE_KEPT_LOG_COMMAND = 'docker builder prune, docker buildx prune';
|
|
|
|
public int $jobTries = 1;
|
|
|
|
/**
|
|
* Bounds how long a lost queued run (for example a killed worker) blocks new cleanups.
|
|
*/
|
|
public int $jobUniqueFor = self::QUEUED_DELAY + 240;
|
|
|
|
public int $jobTimeout = self::JOB_TIMEOUT;
|
|
|
|
private int $deadline = 0;
|
|
|
|
/**
|
|
* Overlap lock key shared by DockerCleanupJob and queued CleanupDocker runs.
|
|
*/
|
|
public static function overlapLockKey(Server $server): string
|
|
{
|
|
return 'docker-cleanup-'.$server->uuid;
|
|
}
|
|
|
|
/**
|
|
* Lock lifetime: longer than the job timeout, so a new run cannot start while a timed-out
|
|
* run is still finishing.
|
|
*/
|
|
public static function overlapLockExpiresAfter(): int
|
|
{
|
|
return self::JOB_TIMEOUT + 60;
|
|
}
|
|
|
|
/**
|
|
* At most one queued cleanup per server. The unique lock is released when a worker starts
|
|
* the job, so a stop during a running cleanup queues one follow-up cleanup.
|
|
*/
|
|
public static function makeUniqueJob(mixed ...$arguments): UniqueJobDecorator
|
|
{
|
|
return new UniqueUntilProcessingJobDecorator(static::class, ...$arguments);
|
|
}
|
|
|
|
public static function lastRunCacheKey(Server $server): string
|
|
{
|
|
return 'docker-cleanup-last-run-'.$server->uuid;
|
|
}
|
|
|
|
/**
|
|
* The deployment that is queued or running on the server (as target or build server), if any.
|
|
* Pruning the build caches fails a build that uses them, so the cleanup keeps them while such a
|
|
* deployment exists. A deployment that wrote no log for longer than the deployment timeout is
|
|
* stuck and does not count.
|
|
*/
|
|
public static function activeDeployment(Server $server): ?ApplicationDeploymentQueue
|
|
{
|
|
return ApplicationDeploymentQueue::query()
|
|
->where(fn ($query) => $query->where('server_id', $server->id)->orWhere('build_server_id', $server->id))
|
|
->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value])
|
|
->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600))
|
|
->first(['id', 'deployment_uuid']);
|
|
}
|
|
|
|
/**
|
|
* The reason the build cache was kept, read from a cleanup log, if it was kept.
|
|
*/
|
|
public static function buildCacheKeptReason(array $cleanupLog): ?string
|
|
{
|
|
return collect($cleanupLog)->firstWhere('command', self::BUILD_CACHE_KEPT_LOG_COMMAND)['output'] ?? null;
|
|
}
|
|
|
|
/**
|
|
* Cleanup after a resource stop: skipped when a cleanup ran recently, and the queued run
|
|
* only cleans when the disk usage is at or above the server cleanup threshold.
|
|
*/
|
|
public static function dispatchAfterStop(Server $server): void
|
|
{
|
|
if (Cache::has(self::lastRunCacheKey($server))) {
|
|
return;
|
|
}
|
|
|
|
static::dispatch($server, false, false, true);
|
|
}
|
|
|
|
public function getJobUniqueId(Server $server): string
|
|
{
|
|
return $server->uuid;
|
|
}
|
|
|
|
public function configureJob(JobDecorator $job): void
|
|
{
|
|
$job->onQueue(maintenance_queue())->delay(self::QUEUED_DELAY);
|
|
}
|
|
|
|
/**
|
|
* Only applies when dispatched as a job. ::run() inside DockerCleanupJob already holds
|
|
* the lock and does not go through this middleware.
|
|
*/
|
|
public function getJobMiddleware(Server $server): array
|
|
{
|
|
return [
|
|
(new WithoutOverlapping(self::overlapLockKey($server)))
|
|
->shared()
|
|
->expireAfter(self::overlapLockExpiresAfter())
|
|
->dontRelease(),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param bool $pruneBuildCacheDuringDeployment Also prune the build caches while a deployment is active
|
|
* (a manual cleanup). Otherwise they stay until the next run.
|
|
*/
|
|
public function handle(Server $server, bool $deleteUnusedVolumes = false, bool $deleteUnusedNetworks = false, bool $onlyAboveThreshold = false, bool $pruneBuildCacheDuringDeployment = false)
|
|
{
|
|
$this->deadline = now()->getTimestamp() + self::REMOTE_COMMANDS_DEADLINE;
|
|
|
|
if ($onlyAboveThreshold) {
|
|
$diskUsage = $server->getDiskUsage();
|
|
if (is_numeric($diskUsage) && (int) $diskUsage < $server->settings->docker_cleanup_threshold) {
|
|
return [];
|
|
}
|
|
}
|
|
$helperImageVersion = getHelperVersion();
|
|
$helperImageWithVersion = coolifyHelperImage().":$helperImageVersion";
|
|
|
|
$cleanupLog = [];
|
|
|
|
// Get all application image repositories to exclude from prune
|
|
$applications = $server->applications();
|
|
$applicationImageRepos = collect($applications)->map(function ($app) {
|
|
return $app->docker_registry_image_name ?? $app->uuid;
|
|
})->unique()->values();
|
|
|
|
// Clean up old application images while preserving N most recent for rollback
|
|
$applicationCleanupLog = $this->cleanupApplicationImages($server, $applications);
|
|
$cleanupLog = array_merge($cleanupLog, $applicationCleanupLog);
|
|
|
|
// Build image prune command that excludes application images and current Coolify infrastructure images
|
|
// This ensures we clean up non-Coolify images while preserving rollback images and current helper image
|
|
// Note: Only the current version is protected; old versions will be cleaned up by explicit commands below
|
|
// We pass the version strings so all registry variants are protected (ghcr.io, docker.io, no prefix)
|
|
$imagePruneCmd = $this->buildImagePruneCommand(
|
|
$applicationImageRepos,
|
|
$helperImageVersion
|
|
);
|
|
|
|
$commands = [
|
|
$this->buildContainerPruneCommand(),
|
|
$imagePruneCmd,
|
|
];
|
|
|
|
$activeDeployment = $pruneBuildCacheDuringDeployment ? null : self::activeDeployment($server);
|
|
if (! $activeDeployment) {
|
|
$commands[] = 'docker builder prune -af';
|
|
// The prune fails on a stopped builder, so start it first.
|
|
$commands[] = railpackBuilderHelperCommand(railpackBuildxMetadataVolume($server), $helperImageWithVersion, 'docker buildx inspect --bootstrap coolify-railpack >/dev/null 2>&1; docker buildx prune --builder coolify-railpack -af');
|
|
} else {
|
|
$cleanupLog[] = [
|
|
'command' => self::BUILD_CACHE_KEPT_LOG_COMMAND,
|
|
'output' => 'Skipped: the build cache was not pruned because deployment '.$activeDeployment->deployment_uuid.' is queued or running on this server.',
|
|
];
|
|
}
|
|
|
|
$commands[] = $this->buildCoolifyImageCleanupCommand($helperImageVersion);
|
|
|
|
if ($deleteUnusedVolumes) {
|
|
$commands[] = 'docker volume prune -af';
|
|
}
|
|
|
|
if ($deleteUnusedNetworks) {
|
|
$commands[] = 'docker network prune -f';
|
|
}
|
|
|
|
foreach ($commands as $command) {
|
|
$commandOutput = $this->runRemoteCommand($command, $server);
|
|
if ($commandOutput !== null) {
|
|
$cleanupLog[] = [
|
|
'command' => $command,
|
|
'output' => remove_iip($commandOutput),
|
|
];
|
|
}
|
|
}
|
|
|
|
Cache::put(self::lastRunCacheKey($server), true, self::STOP_CLEANUP_COOLDOWN);
|
|
|
|
return $cleanupLog;
|
|
}
|
|
|
|
/**
|
|
* Run one remote command with the time left before the cleanup deadline as its local
|
|
* SSH timeout. Errors and timeouts of a single command are ignored, as before; once the
|
|
* deadline has passed the cleanup stops and fails.
|
|
*/
|
|
private function runRemoteCommand(string $command, Server $server): ?string
|
|
{
|
|
$remaining = $this->deadline - now()->getTimestamp();
|
|
if ($remaining < 1) {
|
|
throw new RuntimeException('Docker cleanup did not finish within '.self::REMOTE_COMMANDS_DEADLINE.' seconds. The remaining cleanup commands were not run.');
|
|
}
|
|
|
|
return instant_remote_process([$command], $server, false, timeout: $remaining);
|
|
}
|
|
|
|
/**
|
|
* Build a command that removes stopped Coolify containers, except the proxy and the
|
|
* containers of databases, applications and services.
|
|
*
|
|
* docker container prune only excludes a container that matches all of its label!= filters
|
|
* together, so we list the stopped containers and filter them by label in the shell.
|
|
*/
|
|
private function buildContainerPruneCommand(): string
|
|
{
|
|
return "docker ps -a --filter status=created --filter status=exited --filter status=dead --filter label=coolify.managed=true --format '{{.ID}} coolify.proxy={{.Label \"coolify.proxy\"}} coolify.type={{.Label \"coolify.type\"}}' | ".
|
|
"grep -v -E 'coolify\.proxy=true |coolify\.type=(database|application|service)$' | ".
|
|
"awk '{print $1}' | xargs -r docker rm";
|
|
}
|
|
|
|
/**
|
|
* Build a command that removes unused images of the Coolify helper, realtime and sentinel,
|
|
* from any registry. They have the coolify.managed=true label, so the generic image prune
|
|
* skips them. The current helper version stays, and docker rmi (without -f) keeps images
|
|
* that a container uses, for example the running sentinel.
|
|
*/
|
|
private function buildCoolifyImageCleanupCommand(string $helperImageVersion): string
|
|
{
|
|
$escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion);
|
|
|
|
return "docker images --format '{{.Repository}}:{{.Tag}}' | ".
|
|
"grep -E '(^|/)coollabsio/(coolify-helper|coolify-realtime|sentinel):' | ".
|
|
"grep -v -E ':<none>$|(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$' | ".
|
|
'xargs -r docker rmi 2>/dev/null || true';
|
|
}
|
|
|
|
/**
|
|
* Build a docker image prune command that excludes application image repositories.
|
|
*
|
|
* Since docker image prune doesn't support excluding by repository name directly,
|
|
* we use a shell script approach to delete unused images while preserving application images.
|
|
*/
|
|
private function buildImagePruneCommand(
|
|
$applicationImageRepos,
|
|
string $helperImageVersion
|
|
): string {
|
|
// Step 1: Always prune dangling images (untagged)
|
|
$commands = ['docker image prune -f'];
|
|
|
|
// Build grep pattern to exclude application image repositories (matches repo:tag and repo_service:tag)
|
|
$appExcludePatterns = $applicationImageRepos->map(function ($repo) {
|
|
// Escape special characters for grep extended regex (ERE)
|
|
// ERE special chars: . \ + * ? [ ^ ] $ ( ) { } |
|
|
return preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $repo);
|
|
})->implode('|');
|
|
|
|
// Build grep pattern to exclude Coolify infrastructure images (current version only)
|
|
// This pattern matches the image name regardless of registry prefix:
|
|
// - ghcr.io/coollabsio/coolify-helper:1.0.12
|
|
// - docker.io/coollabsio/coolify-helper:1.0.12
|
|
// - coollabsio/coolify-helper:1.0.12
|
|
// Pattern: (^|/)coollabsio/coolify-helper:VERSION$
|
|
$escapedHelperVersion = preg_replace('/([.\\\\+*?\[\]^$(){}|])/', '\\\\$1', $helperImageVersion);
|
|
$infraExcludePattern = "(^|/)coollabsio/coolify-helper:{$escapedHelperVersion}$";
|
|
|
|
// Delete unused images that:
|
|
// - Are not application images (don't match app repos)
|
|
// - Are not current Coolify helper image (any registry)
|
|
// - Don't have coolify.managed=true label
|
|
// Images in use by containers will fail silently with docker rmi
|
|
// Pattern matches both uuid:tag and uuid_servicename:tag (Docker Compose with build)
|
|
// Images pulled by digest have no tag and are not dangling, so they are listed as repo@digest
|
|
$grepCommands = "grep -v '<none>'";
|
|
|
|
// Add application repo exclusion if there are applications
|
|
if ($applicationImageRepos->isNotEmpty()) {
|
|
// The repository names come from application data, so quote the pattern for the shell.
|
|
$grepCommands .= ' | grep -v -E '.escapeshellarg("^({$appExcludePatterns})[_:@].+");
|
|
}
|
|
|
|
// Add infrastructure image exclusion (matches any registry prefix)
|
|
$grepCommands .= ' | grep -v -E '.escapeshellarg($infraExcludePattern);
|
|
|
|
$commands[] = "docker images --digests --format '{{if eq .Tag \"<none>\"}}{{.Repository}}@{{.Digest}}{{else}}{{.Repository}}:{{.Tag}}{{end}}' | ".
|
|
$grepCommands.' | '.
|
|
"xargs -r -I {} sh -c 'docker inspect --format \"{{index .Config.Labels \\\"coolify.managed\\\"}}\" \"{}\" 2>/dev/null | grep -q true || docker rmi \"{}\" 2>/dev/null' || true";
|
|
|
|
return implode(' && ', $commands);
|
|
}
|
|
|
|
/**
|
|
* Remove old application images while the N most recent images of each application stay
|
|
* for rollback. Uses a fixed number of remote commands, independent of the number of
|
|
* applications on the server.
|
|
*/
|
|
private function cleanupApplicationImages(Server $server, $applications = null): array
|
|
{
|
|
$cleanupLog = [];
|
|
|
|
if ($applications === null) {
|
|
$applications = $server->applications();
|
|
}
|
|
if ($applications->isEmpty()) {
|
|
return $cleanupLog;
|
|
}
|
|
$applications->loadMissing('settings');
|
|
|
|
$disableRetention = $server->settings->disable_application_image_retention ?? false;
|
|
|
|
// Images that a container uses, running or stopped
|
|
$containerImages = collect(explode("\n", $this->runRemoteCommand("docker ps -a --format '{{.Names}}#{{.Image}}' 2>/dev/null || true", $server) ?? ''))
|
|
->filter()
|
|
->map(fn (string $line) => array_pad(explode('#', $line, 2), 2, '')[1])
|
|
->unique();
|
|
|
|
$serverImages = collect(explode("\n", $this->runRemoteCommand("docker images --format '{{.Repository}}#{{.Tag}}#{{.CreatedAt}}' 2>/dev/null || true", $server) ?? ''))
|
|
->filter()
|
|
->map(function (string $line) {
|
|
[$repository, $tag, $createdAt] = array_pad(explode('#', $line, 3), 3, '');
|
|
|
|
return [
|
|
'repository' => $repository,
|
|
'tag' => $tag,
|
|
'created_at' => $createdAt,
|
|
'image_ref' => "{$repository}:{$tag}",
|
|
];
|
|
})
|
|
->filter(fn ($image) => $image['tag'] !== '' && $image['tag'] !== '<none>');
|
|
|
|
// A deployment starts the containers of its new images only after the build or pull,
|
|
// so the images of an application with a queued or running deployment all stay.
|
|
$deployingApplicationIds = ApplicationDeploymentQueue::query()
|
|
->whereIn('application_id', $applications->map(fn ($application) => (string) $application->id))
|
|
->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value])
|
|
->where('updated_at', '>=', now()->subSeconds($server->settings->dynamic_timeout ?? 3600))
|
|
->pluck('application_id');
|
|
|
|
$imagesToDelete = collect();
|
|
|
|
foreach ($applications as $application) {
|
|
if ($deployingApplicationIds->contains((string) $application->id)) {
|
|
continue;
|
|
}
|
|
|
|
$imagesToKeep = $disableRetention ? 0 : ($application->settings->docker_images_to_keep ?? 2);
|
|
$imageRepository = $application->docker_registry_image_name ?? $application->uuid;
|
|
|
|
// Matches both uuid:tag and uuid_servicename:tag (Docker Compose with build), like
|
|
// the docker images --filter reference='<repository>*' filter. Each repository (the
|
|
// application, or one Docker Compose service) has its own retention.
|
|
$repositories = $serverImages
|
|
->filter(fn ($image) => str_starts_with($image['repository'], $imageRepository)
|
|
&& ! str_contains(substr($image['repository'], strlen($imageRepository)), '/'))
|
|
->groupBy('repository');
|
|
|
|
foreach ($repositories as $images) {
|
|
// Tags that a container uses
|
|
$currentTags = $images->filter(fn ($image) => $containerImages->contains($image['image_ref']))->pluck('tag');
|
|
|
|
// Separate images into categories
|
|
// PR images (pr-*) are always deleted
|
|
// Build images (*-build) are cleaned up to match retained regular images
|
|
$prImages = $images->filter(fn ($image) => str_starts_with($image['tag'], 'pr-'));
|
|
$buildImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && str_ends_with($image['tag'], '-build'));
|
|
$regularImages = $images->filter(fn ($image) => ! str_starts_with($image['tag'], 'pr-') && ! str_ends_with($image['tag'], '-build'));
|
|
|
|
// Filter out the images in use from regular images and sort by creation date
|
|
$sortedRegularImages = $regularImages
|
|
->reject(fn ($image) => $currentTags->contains($image['tag']))
|
|
->sortByDesc('created_at')
|
|
->values();
|
|
|
|
// Clean up build images (-build suffix) that don't correspond to retained regular images
|
|
// Build images are intermediate artifacts (e.g. Nixpacks) not used by running containers.
|
|
$keptTags = $sortedRegularImages->take($imagesToKeep)->pluck('tag')->concat($currentTags);
|
|
|
|
$imagesToDelete = $imagesToDelete
|
|
->concat($prImages)
|
|
->concat($sortedRegularImages->skip($imagesToKeep))
|
|
->concat($buildImages->reject(fn ($image) => $keptTags->contains(preg_replace('/-build$/', '', $image['tag']))));
|
|
}
|
|
}
|
|
|
|
// Images in use fail silently; docker rmi continues with the next image
|
|
foreach ($this->imageRemovalCommands($imagesToDelete->pluck('image_ref')->unique()) as $deleteCommand) {
|
|
$deleteOutput = $this->runRemoteCommand($deleteCommand, $server);
|
|
$cleanupLog[] = [
|
|
'command' => $deleteCommand,
|
|
'output' => remove_iip($deleteOutput ?? 'Images removed or were in use'),
|
|
];
|
|
}
|
|
|
|
return $cleanupLog;
|
|
}
|
|
|
|
/**
|
|
* @param Collection<int, string> $imageRefs
|
|
* @return Collection<int, string>
|
|
*/
|
|
private function imageRemovalCommands(Collection $imageRefs): Collection
|
|
{
|
|
return $imageRefs
|
|
->chunk(100)
|
|
->map(fn (Collection $chunk) => 'docker rmi '.$chunk->map(fn (string $ref) => escapeshellarg($ref))->implode(' ').' 2>/dev/null || true')
|
|
->values();
|
|
}
|
|
}
|