mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-06 15:39:59 +00:00
Andras Bacsai
7956be8833
Scheduled, manual and stop-triggered cleanups share one per-server lock. Stop-triggered cleanups wait 60 seconds and are coalesced per server, so a bulk stop queues one cleanup. Remote commands share a 540 second deadline below the 600 second job timeout, and application images are scanned with a fixed number of remote commands.
353 lines
14 KiB
PHP
353 lines
14 KiB
PHP
<?php
|
|
|
|
use App\Actions\Server\CleanupDocker;
|
|
use App\Jobs\DockerCleanupJob;
|
|
use App\Models\Application;
|
|
use App\Models\DockerCleanupExecution;
|
|
use App\Models\Environment;
|
|
use App\Models\InstanceSettings;
|
|
use App\Models\PrivateKey;
|
|
use App\Models\Project;
|
|
use App\Models\ScheduledJobDelivery;
|
|
use App\Models\Server;
|
|
use App\Models\StandaloneDocker;
|
|
use App\Models\Team;
|
|
use App\Models\User;
|
|
use App\Notifications\Server\DockerCleanupFailed;
|
|
use App\Services\ScheduledJobDeliveryService;
|
|
use App\Support\Actions\UniqueUntilProcessingJobDecorator;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Illuminate\Support\Carbon;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Illuminate\Support\Facades\Notification;
|
|
use Illuminate\Support\Facades\Process;
|
|
use Illuminate\Support\Facades\Queue;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use Lorisleiva\Actions\Decorators\JobDecorator;
|
|
|
|
uses(RefreshDatabase::class);
|
|
|
|
beforeEach(function () {
|
|
Server::flushIdentityMap();
|
|
Storage::fake('ssh-keys');
|
|
Storage::fake('ssh-mux');
|
|
});
|
|
|
|
afterEach(function () {
|
|
Server::flushIdentityMap();
|
|
Carbon::setTestNow();
|
|
});
|
|
|
|
function dockerCleanupReachableServer(): Server
|
|
{
|
|
InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
|
|
$team = Team::factory()->create();
|
|
$server = Server::factory()->create([
|
|
'team_id' => $team->id,
|
|
'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
|
|
]);
|
|
$server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
|
|
|
|
return $server->fresh();
|
|
}
|
|
|
|
/**
|
|
* @return Collection<int, string> The local commands (SSH invocations) that ran.
|
|
*/
|
|
function dockerCleanupFakeProcesses(?Closure $onRun = null): Collection
|
|
{
|
|
$commands = collect();
|
|
Process::fake(function ($process) use ($commands, $onRun) {
|
|
$command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
|
|
$commands->push($command);
|
|
if ($onRun) {
|
|
$onRun($command);
|
|
}
|
|
|
|
return Process::result(output: '');
|
|
});
|
|
|
|
return $commands;
|
|
}
|
|
|
|
it('persists the server id when creating an execution record', function () {
|
|
$user = User::factory()->create();
|
|
$team = $user->teams()->first();
|
|
$server = Server::factory()->create(['team_id' => $team->id]);
|
|
|
|
$execution = DockerCleanupExecution::create([
|
|
'server_id' => $server->id,
|
|
]);
|
|
|
|
expect($execution->server_id)->toBe($server->id);
|
|
$this->assertDatabaseHas('docker_cleanup_executions', [
|
|
'id' => $execution->id,
|
|
'server_id' => $server->id,
|
|
]);
|
|
});
|
|
|
|
it('creates a failed execution record when server is not functional', function () {
|
|
$user = User::factory()->create();
|
|
$team = $user->teams()->first();
|
|
$server = Server::factory()->create(['team_id' => $team->id]);
|
|
|
|
// Make server not functional by setting is_reachable to false
|
|
$server->settings->update(['is_reachable' => false]);
|
|
|
|
$job = new DockerCleanupJob($server);
|
|
$job->handle();
|
|
|
|
$execution = DockerCleanupExecution::where('server_id', $server->id)->first();
|
|
|
|
expect($execution)->not->toBeNull()
|
|
->and($execution->status)->toBe('failed')
|
|
->and($execution->message)->toContain('not functional')
|
|
->and($execution->finished_at)->not->toBeNull();
|
|
});
|
|
|
|
it('creates a failed execution record when server is force disabled', function () {
|
|
$user = User::factory()->create();
|
|
$team = $user->teams()->first();
|
|
$server = Server::factory()->create(['team_id' => $team->id]);
|
|
|
|
// Make server not functional by force disabling
|
|
$server->settings->update([
|
|
'is_reachable' => true,
|
|
'is_usable' => true,
|
|
'force_disabled' => true,
|
|
]);
|
|
|
|
$job = new DockerCleanupJob($server);
|
|
$job->handle();
|
|
|
|
$execution = DockerCleanupExecution::where('server_id', $server->id)->first();
|
|
|
|
expect($execution)->not->toBeNull()
|
|
->and($execution->status)->toBe('failed')
|
|
->and($execution->message)->toContain('not functional');
|
|
});
|
|
|
|
it('finishes the latest running execution when the job fails after a timeout', function () {
|
|
$user = User::factory()->create();
|
|
$team = $user->teams()->first();
|
|
$server = Server::factory()->create(['team_id' => $team->id]);
|
|
|
|
$olderExecution = DockerCleanupExecution::create([
|
|
'server_id' => $server->id,
|
|
]);
|
|
$timedOutExecution = DockerCleanupExecution::create([
|
|
'server_id' => $server->id,
|
|
]);
|
|
|
|
$job = new DockerCleanupJob($server);
|
|
$job->failed(new RuntimeException('Docker cleanup job has timed out.'));
|
|
|
|
expect($timedOutExecution->refresh()->status)->toBe('failed')
|
|
->and($timedOutExecution->message)->toBe('Docker cleanup job has timed out.')
|
|
->and($timedOutExecution->finished_at)->not->toBeNull()
|
|
->and($olderExecution->refresh()->status)->toBe('running')
|
|
->and($olderExecution->finished_at)->toBeNull();
|
|
});
|
|
|
|
it('skips a scheduled cleanup occurrence while another cleanup holds the server lock, so it is not reported missed later', function () {
|
|
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
|
|
Notification::fake();
|
|
$server = dockerCleanupReachableServer();
|
|
$occurrence = ScheduledJobDelivery::create([
|
|
'schedule_key' => "docker-cleanup:{$server->id}",
|
|
'scheduled_for' => now()->startOfMinute(),
|
|
'job_type' => 'docker-cleanup',
|
|
'resource_id' => $server->id,
|
|
'status' => 'enqueued',
|
|
'enqueued_at' => now(),
|
|
]);
|
|
$job = new DockerCleanupJob($server, false, false, false, $occurrence->uuid);
|
|
$runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
|
|
$runningCleanupLock->get();
|
|
$commands = dockerCleanupFakeProcesses();
|
|
|
|
dispatch_sync($job);
|
|
$runningCleanupLock->release();
|
|
|
|
expect($occurrence->fresh()->status)->toBe('skipped')
|
|
->and(DockerCleanupExecution::query()->exists())->toBeFalse()
|
|
->and($commands)->toBeEmpty();
|
|
|
|
Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1));
|
|
Queue::fake();
|
|
app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
|
|
|
|
Queue::assertNothingPushed();
|
|
expect(DockerCleanupExecution::query()->exists())->toBeFalse();
|
|
Notification::assertNotSentTo($server->team, DockerCleanupFailed::class);
|
|
});
|
|
|
|
it('drops a manual cleanup while another cleanup holds the server lock without touching occurrences', function () {
|
|
$server = dockerCleanupReachableServer();
|
|
$job = new DockerCleanupJob($server, true);
|
|
$runningCleanupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
|
|
$runningCleanupLock->get();
|
|
$commands = dockerCleanupFakeProcesses();
|
|
|
|
dispatch_sync($job);
|
|
$runningCleanupLock->release();
|
|
|
|
expect(DockerCleanupExecution::query()->exists())->toBeFalse()
|
|
->and(ScheduledJobDelivery::query()->exists())->toBeFalse()
|
|
->and($commands)->toBeEmpty();
|
|
});
|
|
|
|
it('shares one server lock between scheduled cleanups and queued CleanupDocker runs that outlives the job timeout', function () {
|
|
$server = dockerCleanupReachableServer();
|
|
$cleanupJob = new DockerCleanupJob($server);
|
|
$actionJob = CleanupDocker::makeJob($server, false, false);
|
|
|
|
$cleanupLock = $cleanupJob->middleware()[0];
|
|
$actionLock = $actionJob->middleware()[0];
|
|
|
|
expect($cleanupLock->getLockKey($cleanupJob))
|
|
->toBe('laravel-queue-overlap:docker-cleanup-'.$server->uuid)
|
|
->toBe($actionLock->getLockKey($actionJob))
|
|
->and($cleanupLock->expiresAfter)->toBeGreaterThan($cleanupJob->timeout)
|
|
->and($actionLock->expiresAfter)->toBeGreaterThan($actionJob->timeout)
|
|
->and($cleanupLock->releaseAfter)->toBeNull()
|
|
->and($actionLock->releaseAfter)->toBeNull()
|
|
->and($actionJob->timeout)->toBe(CleanupDocker::JOB_TIMEOUT)
|
|
->and($actionJob->tries)->toBe(1);
|
|
});
|
|
|
|
it('drops a queued CleanupDocker run while a scheduled cleanup holds the server lock', function () {
|
|
$server = dockerCleanupReachableServer();
|
|
$actionJob = CleanupDocker::makeJob($server, false, false);
|
|
$runningCleanupLock = Cache::lock('laravel-queue-overlap:docker-cleanup-'.$server->uuid, 3600);
|
|
$runningCleanupLock->get();
|
|
$commands = dockerCleanupFakeProcesses();
|
|
|
|
dispatch_sync($actionJob);
|
|
$runningCleanupLock->release();
|
|
|
|
expect($commands)->toBeEmpty();
|
|
});
|
|
|
|
it('routes docker cleanups to the maintenance queue on cloud and to high on self-hosted', function (bool $selfHosted, string $queue) {
|
|
config(['constants.coolify.self_hosted' => $selfHosted]);
|
|
Queue::fake();
|
|
$server = dockerCleanupReachableServer();
|
|
|
|
DockerCleanupJob::dispatch($server, true);
|
|
CleanupDocker::dispatch($server, false, false);
|
|
|
|
expect(maintenance_queue())->toBe($queue);
|
|
Queue::assertPushedOn($queue, DockerCleanupJob::class);
|
|
Queue::assertPushedOn($queue, UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->decorates(CleanupDocker::class));
|
|
})->with([
|
|
'cloud' => [false, 'maintenance'],
|
|
'self-hosted' => [true, 'high'],
|
|
]);
|
|
|
|
it('gives remote cleanup commands a local timeout and fails once the cleanup deadline has passed', function () {
|
|
Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
|
|
$server = dockerCleanupReachableServer();
|
|
$commands = dockerCleanupFakeProcesses(function (string $command) {
|
|
if (str_contains($command, 'docker ')) {
|
|
Carbon::setTestNow(now()->addSeconds(300));
|
|
}
|
|
});
|
|
|
|
expect(fn () => CleanupDocker::run($server))
|
|
->toThrow(RuntimeException::class, 'Docker cleanup did not finish within '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' seconds');
|
|
|
|
$sshCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker container prune') || str_contains($command, 'docker image prune'))->values();
|
|
|
|
expect(CleanupDocker::REMOTE_COMMANDS_DEADLINE)->toBeLessThan(CleanupDocker::JOB_TIMEOUT)
|
|
->and($sshCommands)->toHaveCount(2)
|
|
->and($sshCommands[0])->toContain('timeout '.CleanupDocker::REMOTE_COMMANDS_DEADLINE.' ssh')
|
|
->and($sshCommands[1])->toContain('timeout '.(CleanupDocker::REMOTE_COMMANDS_DEADLINE - 300).' ssh')
|
|
->and($commands->contains(fn (string $command) => str_contains($command, 'docker builder prune')))->toBeFalse();
|
|
});
|
|
|
|
it('coalesces queued cleanups per server and delays them until a bulk stop is over', function () {
|
|
Queue::fake();
|
|
$first = dockerCleanupReachableServer();
|
|
$second = dockerCleanupReachableServer();
|
|
|
|
foreach (range(1, 30) as $stop) {
|
|
CleanupDocker::dispatch($first, false, false);
|
|
}
|
|
CleanupDocker::dispatch($second, false, false);
|
|
|
|
Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2);
|
|
Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, fn (JobDecorator $job) => $job->delay === CleanupDocker::QUEUED_DELAY);
|
|
});
|
|
|
|
it('accepts a new queued cleanup once the waiting one is lost', function () {
|
|
Queue::fake();
|
|
$server = dockerCleanupReachableServer();
|
|
|
|
CleanupDocker::dispatch($server, false, false);
|
|
$this->travel(CleanupDocker::QUEUED_DELAY + 241)->seconds();
|
|
CleanupDocker::dispatch($server, false, false);
|
|
|
|
Queue::assertPushed(UniqueUntilProcessingJobDecorator::class, 2);
|
|
});
|
|
|
|
it('scans application images with a fixed number of remote commands and keeps the retention rules', function () {
|
|
$server = dockerCleanupReachableServer();
|
|
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
|
|
$project = Project::factory()->create(['team_id' => $server->team_id]);
|
|
$environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]);
|
|
$applications = collect(range(1, 5))->map(fn () => Application::factory()->create([
|
|
'environment_id' => $environment->id,
|
|
'destination_id' => $destination->id,
|
|
'destination_type' => $destination->getMorphClass(),
|
|
]));
|
|
$app = $applications->first();
|
|
$app->settings()->update(['docker_images_to_keep' => 1]);
|
|
|
|
$containers = "{$app->uuid}#{$app->uuid}:v2";
|
|
$images = implode("\n", [
|
|
"{$app->uuid}#v1#2026-01-01 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#v2#2026-01-02 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#v3#2026-01-03 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#v4#2026-01-04 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#pr-7#2026-01-05 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#v1-build#2026-01-01 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#v2-build#2026-01-02 00:00:00 +0000 UTC",
|
|
"{$app->uuid}#v4-build#2026-01-04 00:00:00 +0000 UTC",
|
|
"{$app->uuid}_web#v1#2026-01-01 00:00:00 +0000 UTC",
|
|
"{$app->uuid}/other#v1#2026-01-01 00:00:00 +0000 UTC",
|
|
]);
|
|
|
|
$commands = collect();
|
|
Process::fake(function ($process) use ($commands, $containers, $images) {
|
|
$command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
|
|
$commands->push($command);
|
|
|
|
return match (true) {
|
|
str_contains($command, 'docker ps -a --format') => Process::result(output: $containers),
|
|
str_contains($command, "docker images --format '{{.Repository}}#") => Process::result(output: $images),
|
|
default => Process::result(output: ''),
|
|
};
|
|
});
|
|
|
|
CleanupDocker::run($server);
|
|
|
|
$applicationScanCommands = $commands->filter(fn (string $command) => str_contains($command, 'docker ps -a --format')
|
|
|| str_contains($command, "docker images --format '{{.Repository}}#")
|
|
|| str_contains($command, 'docker inspect --format=')
|
|
|| str_contains($command, "--filter reference='"));
|
|
$removals = $commands->filter(fn (string $command) => str_contains($command, 'docker rmi \''))->values();
|
|
|
|
expect($applicationScanCommands)->toHaveCount(2)
|
|
->and($removals)->toHaveCount(1);
|
|
|
|
$removal = $removals->first();
|
|
foreach (["{$app->uuid}:v1", "{$app->uuid}:v3", "{$app->uuid}:pr-7", "{$app->uuid}:v1-build", "{$app->uuid}_web:v1"] as $deleted) {
|
|
expect($removal)->toContain("'{$deleted}'");
|
|
}
|
|
foreach (["{$app->uuid}:v2'", "{$app->uuid}:v4'", "{$app->uuid}:v2-build", "{$app->uuid}:v4-build", "{$app->uuid}/other"] as $kept) {
|
|
expect($removal)->not->toContain($kept);
|
|
}
|
|
});
|