coolify/tests/Feature/DeleteResourceJobAtomicityTest.php

<?php

use App\Actions\Service\DeleteService;
use App\Actions\Service\StopService;
use App\Jobs\DeleteResourceJob;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Notifications\Internal\GeneralNotification;
use Illuminate\Foundation\Console\QueuedCommand;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;

uses(RefreshDatabase::class);

beforeEach(function () {
    Server::flushIdentityMap();
    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));

    $team = Team::factory()->create();
    $server = Server::factory()->create(['team_id' => $team->id]);
    $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
    $project = Project::factory()->create(['team_id' => $team->id]);
    $environment = $project->environments()->first()
        ?? Environment::factory()->create(['project_id' => $project->id]);

    $this->application = Application::factory()->create([
        'environment_id' => $environment->id,
        'destination_id' => $destination->id,
        'destination_type' => $destination->getMorphClass(),
    ]);
    $this->storage = $this->application->persistentStorages()->create([
        'name' => 'delete-resource-job-test',
        'mount_path' => '/data',
        'host_path' => null,
    ]);

    $this->application->delete();
    Queue::fake();
});

it('deletes a non-service Coolify resource when remote cleanup fails', function () {
    Process::fake(['*' => Process::result(errorOutput: 'SSH connection timed out', exitCode: 255)]);

    (new DeleteResourceJob($this->application))->handle();

    expect(Application::withTrashed()->find($this->application->id))->toBeNull();
    Queue::assertNotPushed(QueuedCommand::class);
});

it('removes a service from Coolify and warns the team when its server does not respond', function () {
    Notification::fake();
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    ServiceApplication::create([
        'service_id' => $service->id,
        'name' => 'web',
        'image' => 'nginx:alpine',
    ]);
    $service->server->team->webhookNotificationSettings()->update([
        'webhook_enabled' => true,
        'webhook_url' => 'https://example.com/webhook',
    ]);
    Process::fake();

    (new DeleteResourceJob($service))->handle();

    Process::assertNothingRan();
    expect(Service::withTrashed()->find($service->id))->toBeNull()
        ->and(ServiceApplication::withTrashed()->where('service_id', $service->id)->exists())->toBeFalse();
    Notification::assertSentTo(
        $service->team(),
        GeneralNotification::class,
        fn (GeneralNotification $notification): bool => ! $notification->success
            && str_contains($notification->message, "was removed from Coolify, but its Docker resources could not be removed from server '{$service->server->name}'")
    );
});

it('removes a service from Coolify when its server was deleted', function () {
    Notification::fake();
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    ServiceApplication::create([
        'service_id' => $service->id,
        'name' => 'web',
        'image' => 'nginx:alpine',
    ]);
    // "Delete server" soft-deletes the server before the queued resource jobs run.
    $service->server->delete();
    Server::flushIdentityMap();
    Process::fake();

    (new DeleteResourceJob($service->fresh()))->handle();

    Process::assertNothingRan();
    expect(Service::withTrashed()->find($service->id))->toBeNull()
        ->and(ServiceApplication::withTrashed()->where('service_id', $service->id)->exists())->toBeFalse();
    Notification::assertNothingSent();
});

it('stops a service without an error when its server was deleted', function () {
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    $service->server->delete();
    Server::flushIdentityMap();

    expect(StopService::run($service->fresh()))->toBe('Server is not functional');
});

it('skips the container of a service part when the server does not respond', function () {
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    $part = ServiceApplication::create(['service_id' => $service->id, 'name' => 'web', 'image' => 'nginx:alpine']);
    Process::fake();

    expect(app(DeleteService::class)->removeSubresourceContainer($part))->toBeFalse();
    Process::assertNothingRan();
});

it('deletes only local service metadata when explicitly requested', function () {
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    ServiceApplication::create([
        'service_id' => $service->id,
        'name' => 'web',
        'image' => 'nginx:alpine',
    ]);
    Process::fake();

    (new DeleteResourceJob($service, deleteFromCoolifyOnly: true))->handle();

    Process::assertNothingRan();
    expect(Service::withTrashed()->find($service->id))->toBeNull()
        ->and(ServiceApplication::withTrashed()->where('service_id', $service->id)->exists())->toBeFalse();
});

it('removes service containers before its volumes and local metadata', function () {
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    $application = ServiceApplication::create([
        'service_id' => $service->id,
        'name' => 'web',
        'image' => 'nginx:alpine',
    ]);
    $application->persistentStorages()->create([
        'name' => "{$service->uuid}_web-data",
        'mount_path' => '/data',
        'host_path' => null,
    ]);
    $privateKey = PrivateKey::factory()->create(['team_id' => $service->server->team_id]);
    $service->server->update(['private_key_id' => $privateKey->id]);
    $service->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
    $commands = collect();
    Process::fake(function ($process) use ($commands) {
        $commands->push($process->command);

        return Process::result(output: '');
    });

    (new DeleteResourceJob($service))->handle();

    $commandList = $commands->implode("\n");
    expect($commandList)
        ->toContain("label=coolify.serviceUuid={$service->uuid}")
        ->toContain("label=com.docker.compose.project={$service->uuid}")
        // A numeric id can belong to another Coolify instance on the same server, so it never selects containers.
        ->not->toContain("label=coolify.serviceId={$service->id}")
        ->toContain('sort -u')
        ->toContain('docker rm -f $container_ids')
        ->toContain("docker volume rm -f '{$service->uuid}_web-data'")
        ->and(strpos($commandList, 'docker rm -f $container_ids'))
        ->toBeLessThan(strpos($commandList, 'docker volume rm -f'));
    expect(Service::withTrashed()->find($service->id))->toBeNull();
});

it('targets a service subresource container by its Docker labels', function () {
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    $application = ServiceApplication::create([
        'service_id' => $service->id,
        'name' => 'web',
        'image' => 'nginx:alpine',
    ]);
    $privateKey = PrivateKey::factory()->create(['team_id' => $service->server->team_id]);
    $service->server->update(['private_key_id' => $privateKey->id]);
    $service->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
    $commands = collect();
    Process::fake(function ($process) use ($commands) {
        $commands->push($process->command);

        return Process::result(output: '');
    });

    app(DeleteService::class)->removeSubresourceContainer($application);

    expect($commands->implode("\n"))
        ->toContain("label=coolify.serviceUuid={$service->uuid}")
        ->toContain("label=coolify.service.subUuid={$application->uuid}")
        ->toContain('label=coolify.service.subType=application')
        // Containers from before the UUID labels are found by compose project and service key.
        ->toContain("label=com.docker.compose.project={$service->uuid}")
        ->toContain('label=com.docker.compose.service=web')
        ->toContain('docker rm -f $container_ids');
});

it('removes only the database container when an application of the same service has the same id', function () {
    // Service applications and databases are separate tables, so they can share an id.
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    $application = ServiceApplication::create(['service_id' => $service->id, 'name' => 'web', 'image' => 'nginx:alpine']);
    $database = ServiceDatabase::create(['service_id' => $service->id, 'name' => 'db', 'image' => 'postgres:17-alpine']);
    $database->forceFill(['id' => $application->id])->save();
    $privateKey = PrivateKey::factory()->create(['team_id' => $service->server->team_id]);
    $service->server->update(['private_key_id' => $privateKey->id]);
    $service->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
    $commands = collect();
    Process::fake(function ($process) use ($commands) {
        $commands->push($process->command);

        return Process::result(output: '');
    });

    app(DeleteService::class)->removeSubresourceContainer($database->fresh());

    expect($commands->implode("\n"))
        ->toContain("label=coolify.service.subUuid={$database->uuid}")
        ->toContain('label=coolify.service.subType=database')
        ->not->toContain("label=coolify.service.subUuid={$application->uuid}")
        ->not->toContain('label=coolify.service.subType=application');
});

it('rolls back local metadata deletion when deleting the resource fails', function () {
    Process::fake(['*' => Process::result(output: '')]);
    $applicationUuid = $this->application->uuid;
    Application::deleting(function (Application $application) use ($applicationUuid): void {
        if ($application->uuid === $applicationUuid && $application->isForceDeleting()) {
            throw new RuntimeException('Local deletion failed.');
        }
    });

    expect(fn () => (new DeleteResourceJob($this->application))->handle())
        ->toThrow(RuntimeException::class, 'Local deletion failed.');

    expect($this->storage->fresh())->not->toBeNull()
        ->and(Application::withTrashed()->find($this->application->id))->not->toBeNull();
});

it('deletes scheduled volume backups outside the local deletion transaction', function () {
    $backup = $this->storage->scheduledBackups()->create([
        'team_id' => $this->application->environment->project->team_id,
        'frequency' => 'daily',
        'timeout' => 3600,
    ]);
    $transactionLevel = DB::transactionLevel();
    $deletingTransactionLevel = null;

    ScheduledVolumeBackup::deleting(function (ScheduledVolumeBackup $deletingBackup) use ($backup, &$deletingTransactionLevel): void {
        if ($deletingBackup->is($backup)) {
            $deletingTransactionLevel = DB::transactionLevel();
        }
    });
    Process::fake(['*' => Process::result(output: '')]);

    (new DeleteResourceJob($this->application))->handle();

    expect($deletingTransactionLevel)->toBe($transactionLevel);
});

it('deletes preview metadata locally when its application destination is missing', function () {
    $this->application->restore();
    $this->application->update(['destination_id' => PHP_INT_MAX]);
    $preview = ApplicationPreview::create([
        'uuid' => 'preview-without-destination',
        'application_id' => $this->application->id,
        'pull_request_id' => 45,
        'pull_request_html_url' => 'https://github.com/coollabsio/coolify/pull/45',
    ]);
    $previewStorage = $preview->persistentStorages()->create([
        'name' => 'preview-without-destination-data',
        'mount_path' => '/preview-data',
        'host_path' => null,
    ]);
    Process::fake();

    (new DeleteResourceJob($preview))->handle();

    Process::assertNothingRan();
    expect(ApplicationPreview::withTrashed()->find($preview->id))->toBeNull()
        ->and($previewStorage->fresh())->toBeNull();
});

it('removes service containers with a command that works for non-root SSH users', function () {
    $service = Service::factory()->create([
        'environment_id' => $this->application->environment_id,
        'server_id' => $this->application->destination->server_id,
        'destination_id' => $this->application->destination_id,
        'destination_type' => $this->application->destination_type,
    ]);
    $application = ServiceApplication::create(['service_id' => $service->id, 'name' => 'web', 'image' => 'nginx:alpine']);
    $privateKey = PrivateKey::factory()->create(['team_id' => $service->server->team_id]);
    $service->server->update(['private_key_id' => $privateKey->id, 'user' => 'coolify']);
    $service->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
    $commands = collect();
    Process::fake(function ($process) use ($commands) {
        $commands->push($process->command);

        return Process::result(output: '');
    });

    app(DeleteService::class)->removeSubresourceContainer($application);

    // A leading variable assignment becomes "sudo container_ids=...", which sudo rejects.
    expect($commands->implode("\n"))
        ->not->toContain('sudo container_ids=')
        ->toContain("sudo sh -c '")
        ->toContain("label=coolify.service.subUuid={$application->uuid}")
        ->toContain('docker rm -f $container_ids');
});