fix(notifications): notify when a resource stops unexpectedly

Status checks now queue a delayed check for each application, preview,
service part, or database that goes from up to exited. After 2 minutes
it sends the "Resource status changes" notification if the resource is
still stopped, no deployment runs, and Coolify did not stop it. Each
resource sends at most one notification in 15 minutes.

Closes #9493

CHANGELOG.md

@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ### Fixed
 
+- Send the **Resource status changes** notification when an application, preview, service, or database stops unexpectedly, for example after a crash or `docker kill`. Before, Coolify sent this notification only when it restarted the proxy or a database TCP proxy. Coolify waits 2 minutes and sends the notification only when the resource is still stopped, so a container that Docker restarts, or a deployment that replaces it, sends none. A stop, restart, or redeploy from Coolify sends no notification, and the same resource sends at most one notification in 15 minutes ([#9493](https://github.com/coollabsio/coolify/issues/9493))
 - Fix the "Network error" when you sign in with email in AppFlowy Web. AppFlowy Web runs on its own domain and sends a `Redirect_to` header to GoTrue on the admin domain, and GoTrue blocked the CORS preflight because it did not allow that header. The AppFlowy service template now sets `GOTRUE_CORS_ALLOWED_HEADERS=redirect_to`. For an existing AppFlowy service, add this variable to the `gotrue` service in the Docker Compose file and restart the service ([#10545](https://github.com/coollabsio/coolify/issues/10545))
 - List the commit images of Docker Compose applications on the **Rollback** page and in `GET /api/v1/applications/{uuid}/rollback-images`. Since v4.5.0 the list was always empty, because a Compose build names each service image `<uuid>_<service>`. The page also shows the running commit when the Compose file has services from other images, for example a database ([#12446](https://github.com/coollabsio/coolify/issues/12446))
 - Use PostgreSQL 17 instead of MySQL in the Bugsink service template, as the Bugsink Docker Compose sample does, and pin Bugsink to version 2. Before, the template used `mysql:latest` with binary logging on, so the binary logs filled the disk after some weeks and Bugsink stopped. Only new Bugsink services get PostgreSQL. To stop the growth in an existing service, run `PURGE BINARY LOGS BEFORE NOW();` in MySQL, then add `command: ['--disable-log-bin']` to the `mysql` service and restart it ([#10626](https://github.com/coollabsio/coolify/issues/10626))

app/Actions/Application/StopApplication.php

@@ -5,6 +5,7 @@ namespace App\Actions\Application;
 use App\Actions\Server\CleanupDocker;
 use App\Events\ApplicationStopFailed;
 use App\Events\ServiceStatusChanged;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\Application;
 use Illuminate\Support\Facades\Log;
 use Lorisleiva\Actions\Concerns\AsAction;
@@ -17,6 +18,7 @@ class StopApplication
 
     public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true, bool $removeContainers = true): ?string
     {
+        NotifyResourceStoppedJob::expectStop($application, $application->settings->stopGracePeriodSeconds());
         $containerPresent = ! $removeContainers;
         $servers = collect([$application->destination->server]);
         if ($application?->additional_servers?->count() > 0) {

app/Actions/Application/StopApplicationPreview.php

@@ -3,6 +3,7 @@
 namespace App\Actions\Application;
 
 use App\Events\ServiceStatusChanged;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\ApplicationPreview;
 use Lorisleiva\Actions\Concerns\AsAction;
 
@@ -14,6 +15,7 @@ class StopApplicationPreview
     {
         $application = $preview->application;
         $server = $application->destination->server;
+        NotifyResourceStoppedJob::expectStop($preview, $application->settings->stopGracePeriodSeconds());
         $containers = getCurrentApplicationContainerStatus($server, $application, $preview->pull_request_id);
 
         foreach ($containers->pluck('Names') as $containerName) {

app/Actions/Database/StopDatabase.php

@@ -4,6 +4,7 @@ namespace App\Actions\Database;
 
 use App\Actions\Server\CleanupDocker;
 use App\Events\ServiceStatusChanged;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\BaseModel;
 use App\Models\StandaloneClickhouse;
 use App\Models\StandaloneDragonfly;
@@ -33,6 +34,7 @@ class StopDatabase
                 $removeContainer = false;
             }
 
+            NotifyResourceStoppedJob::expectStop($database);
             $this->stopContainer($database, $database->uuid, 30, $removeContainer);
 
             // Reset restart tracking when database is manually stopped

app/Actions/Docker/GetContainersStatus.php

@@ -9,6 +9,7 @@ use App\Actions\Database\StopDatabaseProxy;
 use App\Actions\Service\StopServiceApplication;
 use App\Actions\Shared\ComplexStatusCheck;
 use App\Events\ServiceChecked;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\Application;
 use App\Models\ApplicationPreview;
 use App\Models\Server;
@@ -125,6 +126,11 @@ class GetContainersStatus implements ShouldBeUnique
         $databases = $this->server->databases();
         $services = $this->server->services()->get();
         $previews = $this->server->previews();
+        $upResourceIds = NotifyResourceStoppedJob::upResourceIds(collect($this->applications)
+            ->concat($previews)
+            ->concat($databases)
+            ->concat($services->flatMap(fn ($service) => $service->applications))
+            ->concat($services->flatMap(fn ($service) => $service->databases)));
         $foundApplications = [];
         $foundApplicationPreviews = [];
         $foundDatabases = [];
@@ -555,6 +561,8 @@ class GetContainersStatus implements ShouldBeUnique
         // Aggregate multi-container service statuses
         $this->aggregateServiceContainerStatuses($services);
 
+        NotifyResourceStoppedJob::dispatchForStopped($upResourceIds);
+
         ServiceChecked::dispatch($this->server->team->id);
     }
 

app/Actions/Service/DeployServiceApplication.php

@@ -4,6 +4,7 @@ namespace App\Actions\Service;
 
 use App\Actions\Shared\EnsureContentFilesOnServer;
 use App\Events\ServiceStartFinished;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
 use Lorisleiva\Actions\Concerns\AsAction;
@@ -19,6 +20,7 @@ class DeployServiceApplication
     {
         $service = $serviceApplication->service;
         $composeServiceName = $serviceApplication->name;
+        NotifyResourceStoppedJob::expectStop($serviceApplication);
 
         $service->parse();
         StartService::queueFileStorageSync($service);

app/Actions/Service/RestartServiceApplication.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions\Service;
 
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
 use Lorisleiva\Actions\Concerns\AsAction;
@@ -21,6 +22,7 @@ class RestartServiceApplication
         $service = $serviceApplication->service;
         $server = $service->destination->server;
         $containerName = escapeshellarg($serviceApplication->name.'-'.$service->uuid);
+        NotifyResourceStoppedJob::expectStop($serviceApplication);
 
         instant_remote_process([
             "docker restart {$containerName}",

app/Actions/Service/StartService.php

@@ -4,6 +4,7 @@ namespace App\Actions\Service;
 
 use App\Actions\Shared\EnsureContentFilesOnServer;
 use App\Events\ServiceStartFinished;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Jobs\ServerFilesFromServerJob;
 use App\Models\LocalFileVolume;
 use App\Models\Service;
@@ -30,6 +31,9 @@ class StartService
         // Fetch remote secrets before stopping: if the secret manager fails, the service keeps running.
         // saveComposeConfigs() below reuses the fetched secrets.
         $service->ensureRemoteSecretsResolvable($service->environment_variables()->get());
+        // Compose recreates the containers, so a status check can find them stopped for a moment.
+        $service->applications()->get()->concat($service->databases()->get())
+            ->each(fn (ServiceApplication|ServiceDatabase $part) => NotifyResourceStoppedJob::expectStop($part));
         if ($this->shouldStopBeforeStarting($pullLatestImages, $stopBeforeStart)) {
             StopService::run(service: $service, dockerCleanup: false);
         }

app/Actions/Service/StopService.php

@@ -5,6 +5,7 @@ namespace App\Actions\Service;
 use App\Actions\Server\CleanupDocker;
 use App\Enums\ProcessStatus;
 use App\Events\ServiceStatusChanged;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\Server;
 use App\Models\Service;
 use Lorisleiva\Actions\Concerns\AsAction;
@@ -39,10 +40,12 @@ class StopService
             $containersToStop = [];
             $applications = $service->applications()->get();
             foreach ($applications as $application) {
+                NotifyResourceStoppedJob::expectStop($application);
                 $containersToStop[] = "{$application->name}-{$service->uuid}";
             }
             $dbs = $service->databases()->get();
             foreach ($dbs as $db) {
+                NotifyResourceStoppedJob::expectStop($db);
                 $containersToStop[] = "{$db->name}-{$service->uuid}";
             }
 

app/Actions/Service/StopServiceApplication.php

@@ -3,6 +3,7 @@
 namespace App\Actions\Service;
 
 use App\Events\ServiceStatusChanged;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
 use Lorisleiva\Actions\Concerns\AsAction;
@@ -18,6 +19,7 @@ class StopServiceApplication
         $service = $serviceApplication->service;
         $server = $service->destination->server;
         $containerName = escapeshellarg($serviceApplication->name.'-'.$service->uuid);
+        NotifyResourceStoppedJob::expectStop($serviceApplication);
 
         if ($removeContainer) {
             $commands = ["docker rm -f {$containerName}"];

app/Jobs/NotifyResourceStoppedJob.php

@@ -0,0 +1,136 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Enums\ApplicationDeploymentStatus;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\ApplicationPreview;
+use App\Models\BaseModel;
+use App\Notifications\Application\StatusChanged;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Cache;
+
+/**
+ * Sends the "resource stopped" notification when a status check finds that a resource stopped
+ * without a stop from Coolify (for example a crash or `docker kill`).
+ *
+ * The job runs after a delay and checks the resource again. A resource that is up again by then
+ * (Docker restarted it, or a deployment replaced its container) sends no notification.
+ */
+class NotifyResourceStoppedJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public const CONFIRMATION_DELAY_SECONDS = 120;
+
+    /**
+     * How long a stop that Coolify starts sends no notification, in addition to the stop timeout.
+     */
+    private const EXPECTED_STOP_SECONDS = 300;
+
+    public $tries = 1;
+
+    public $deleteWhenMissingModels = true;
+
+    public function __construct(public BaseModel $resource)
+    {
+        $this->onQueue('high');
+        $this->delay(now()->addSeconds(self::CONFIRMATION_DELAY_SECONDS));
+    }
+
+    /**
+     * Call before Coolify stops or recreates the containers of a resource, so that the stop sends no notification.
+     */
+    public static function expectStop(BaseModel $resource, int $stopTimeoutSeconds = 30): void
+    {
+        Cache::put(self::expectedStopKey($resource), true, now()->addSeconds($stopTimeoutSeconds + self::EXPECTED_STOP_SECONDS));
+    }
+
+    /**
+     * Read which resources are up, before a status check changes their status.
+     *
+     * @param  iterable<BaseModel>  $resources
+     * @return array<class-string<BaseModel>, list<int>>
+     */
+    public static function upResourceIds(iterable $resources): array
+    {
+        $ids = [];
+        foreach ($resources as $resource) {
+            if (filled($resource->status) && ! self::isExited($resource->status)) {
+                $ids[$resource::class][] = $resource->getKey();
+            }
+        }
+
+        return $ids;
+    }
+
+    /**
+     * After a status check: queue a notification for each resource that was up before the check and is exited now.
+     *
+     * @param  array<class-string<BaseModel>, list<int>>  $upResourceIds  from upResourceIds()
+     */
+    public static function dispatchForStopped(array $upResourceIds): void
+    {
+        foreach ($upResourceIds as $class => $ids) {
+            $class::query()
+                ->whereKey($ids)
+                ->where('status', 'like', 'exited%')
+                ->get()
+                ->reject(fn (BaseModel $resource): bool => self::stopIsExpected($resource))
+                ->each(fn (BaseModel $resource) => self::dispatch($resource));
+        }
+    }
+
+    public function handle(): void
+    {
+        $resource = $this->resource::query()->whereKey($this->resource->getKey())->first();
+        if ($resource === null
+            || ! self::isExited($resource->status)
+            || (bool) $resource->restart_limit_reached
+            || self::stopIsExpected($resource)
+            || self::isDeploying($resource)) {
+            return;
+        }
+
+        $team = $resource instanceof ApplicationPreview ? $resource->application?->team() : $resource->team();
+        $team?->notify(new StatusChanged($resource));
+    }
+
+    private static function isExited(?string $status): bool
+    {
+        return str($status)->startsWith('exited');
+    }
+
+    private static function stopIsExpected(BaseModel $resource): bool
+    {
+        return Cache::has(self::expectedStopKey($resource));
+    }
+
+    private static function expectedStopKey(BaseModel $resource): string
+    {
+        return 'resource-stop-expected:'.$resource->getMorphClass().':'.$resource->getKey();
+    }
+
+    private static function isDeploying(BaseModel $resource): bool
+    {
+        [$applicationId, $pullRequestId] = match (true) {
+            $resource instanceof Application => [$resource->id, 0],
+            $resource instanceof ApplicationPreview => [$resource->application_id, $resource->pull_request_id],
+            default => [null, null],
+        };
+        if ($applicationId === null) {
+            return false;
+        }
+
+        return ApplicationDeploymentQueue::query()
+            ->where('application_id', (string) $applicationId)
+            ->where('pull_request_id', $pullRequestId)
+            ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value])
+            ->exists();
+    }
+}

app/Jobs/PushServerUpdateJob.php

@@ -247,6 +247,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
         $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
         $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();
+        $upResourceIds = NotifyResourceStoppedJob::upResourceIds($this->applications->whereIn('id', $this->allApplicationIds)
+            ->concat($this->previews)
+            ->concat($this->databases)
+            ->concat($this->serviceApplicationsById)
+            ->concat($this->serviceDatabasesById));
 
         // Owners of application containers outside this server's applications, in one query.
         $foreignApplicationIdsByUuid = containerApplicationIdsByUuid(
@@ -405,6 +410,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
         // Aggregate multi-container service statuses
         $this->aggregateServiceContainerStatuses();
 
+        NotifyResourceStoppedJob::dispatchForStopped($upResourceIds);
+
         $this->checkLogDrainContainer();
 
         ServiceChecked::dispatch($this->server->team_id);

app/Notifications/Application/StatusChanged.php

@@ -2,39 +2,62 @@
 
 namespace App\Notifications\Application;
 
+use App\Contracts\ThrottledNotification;
 use App\Models\Application;
+use App\Models\ApplicationPreview;
+use App\Models\BaseModel;
+use App\Models\ServiceApplication;
+use App\Models\ServiceDatabase;
 use App\Notifications\CustomEmailNotification;
 use App\Notifications\Dto\DiscordMessage;
 use App\Notifications\Dto\PushoverMessage;
 use App\Notifications\Dto\SlackMessage;
+use Illuminate\Database\Eloquent\Model;
 use Illuminate\Notifications\Messages\MailMessage;
 
-class StatusChanged extends CustomEmailNotification
+/**
+ * A resource (application, preview, service part, or database) stopped without a stop from Coolify.
+ */
+class StatusChanged extends CustomEmailNotification implements ThrottledNotification
 {
     public string $resource_name;
 
-    public string $project_uuid;
+    public ?string $project_name;
 
-    public string $environment_uuid;
+    public ?string $project_uuid;
 
-    public string $environment_name;
+    public ?string $environment_uuid;
+
+    public ?string $environment_name;
 
     public ?string $resource_url = null;
 
     public ?string $fqdn;
 
-    public function __construct(public Application $resource)
+    public function __construct(public BaseModel $resource)
     {
         $this->onQueue('high');
-        $this->resource_name = data_get($resource, 'name');
-        $this->project_uuid = data_get($resource, 'environment.project.uuid');
-        $this->environment_uuid = data_get($resource, 'environment.uuid');
-        $this->environment_name = data_get($resource, 'environment.name');
+        $environment = data_get($resource, 'environment')
+            ?? data_get($resource, 'application.environment')
+            ?? data_get($resource, 'service.environment');
+        $this->resource_name = $resource instanceof ApplicationPreview
+            ? data_get($resource, 'application.name').' PR #'.$resource->pull_request_id
+            : (string) data_get($resource, 'name');
+        $this->project_name = data_get($environment, 'project.name');
+        $this->project_uuid = data_get($environment, 'project.uuid');
+        $this->environment_uuid = data_get($environment, 'uuid');
+        $this->environment_name = data_get($environment, 'name');
         $this->fqdn = data_get($resource, 'fqdn', null);
         if (str($this->fqdn)->explode(',')->count() > 1) {
             $this->fqdn = str($this->fqdn)->explode(',')->first();
         }
-        $this->resource_url = base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/application/{$this->resource->uuid}";
+        [$type, $uuid] = match (true) {
+            $resource instanceof Application => ['application', $resource->uuid],
+            $resource instanceof ApplicationPreview => ['application', data_get($resource, 'application.uuid')],
+            $resource instanceof ServiceApplication, $resource instanceof ServiceDatabase => ['service', data_get($resource, 'service.uuid')],
+            default => ['database', $resource->uuid],
+        };
+        $this->resource_url = base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/{$type}/{$uuid}";
     }
 
     public function via(object $notifiable): array
@@ -42,14 +65,26 @@ class StatusChanged extends CustomEmailNotification
         return $notifiable->getEnabledChannels('status_change');
     }
 
+    public function throttleSubject(): ?Model
+    {
+        return $this->resource;
+    }
+
+    /**
+     * A resource that stops again soon after it came back sends no second notification.
+     */
+    public function throttleIntervalMinutes(): int
+    {
+        return 15;
+    }
+
     public function toMail(): MailMessage
     {
         $mail = new MailMessage;
-        $fqdn = $this->fqdn;
-        $mail->subject("Coolify: {$this->resource_name} has been stopped");
+        $mail->subject("Coolify: {$this->resource_name} has stopped");
         $mail->view('emails.application-status-changes', [
             'name' => $this->resource_name,
-            'fqdn' => $fqdn,
+            'fqdn' => $this->fqdn,
             'resource_url' => $this->resource_url,
         ]);
 
@@ -59,8 +94,8 @@ class StatusChanged extends CustomEmailNotification
     public function toDiscord(): DiscordMessage
     {
         return new DiscordMessage(
-            title: ':cross_mark: Application stopped',
-            description: '[Open Application in Coolify]('.$this->resource_url.')',
+            title: ':cross_mark: Resource stopped',
+            description: "{$this->resource_name} has stopped.\n\n[Open Resource in Coolify]({$this->resource_url})",
             color: DiscordMessage::errorColor(),
             isCritical: true,
         );
@@ -68,13 +103,11 @@ class StatusChanged extends CustomEmailNotification
 
     public function toTelegram(): array
     {
-        $message = 'Coolify: '.$this->resource_name.' has been stopped.';
-
         return [
-            'message' => $message,
+            'message' => "Coolify: {$this->resource_name} has stopped.",
             'buttons' => [
                 [
-                    'text' => 'Open Application in Coolify',
+                    'text' => 'Open Resource in Coolify',
                     'url' => $this->resource_url,
                 ],
             ],
@@ -83,15 +116,13 @@ class StatusChanged extends CustomEmailNotification
 
     public function toPushover(): PushoverMessage
     {
-        $message = $this->resource_name.' has been stopped.';
-
         return new PushoverMessage(
-            title: 'Application stopped',
+            title: 'Resource stopped',
             level: 'error',
-            message: $message,
+            message: "{$this->resource_name} has stopped.",
             buttons: [
                 [
-                    'text' => 'Open Application in Coolify',
+                    'text' => 'Open Resource in Coolify',
                     'url' => $this->resource_url,
                 ],
             ],
@@ -100,15 +131,13 @@ class StatusChanged extends CustomEmailNotification
 
     public function toSlack(): SlackMessage
     {
-        $title = 'Application stopped';
-        $description = "{$this->resource_name} has been stopped";
-
-        $description .= "\n\n*Project:* ".data_get($this->resource, 'environment.project.name');
+        $description = "{$this->resource_name} has stopped";
+        $description .= "\n\n*Project:* {$this->project_name}";
         $description .= "\n*Environment:* {$this->environment_name}";
-        $description .= "\n*Application URL:* {$this->resource_url}";
+        $description .= "\n*Resource URL:* {$this->resource_url}";
 
         return new SlackMessage(
-            title: $title,
+            title: 'Resource stopped',
             description: $description,
             color: SlackMessage::errorColor()
         );
@@ -118,12 +147,15 @@ class StatusChanged extends CustomEmailNotification
     {
         return [
             'success' => false,
-            'message' => 'Application stopped',
+            'message' => 'Resource stopped',
             'event' => 'status_changed',
+            'status' => 'exited',
             'application_name' => $this->resource_name,
             'application_uuid' => $this->resource->uuid,
+            'resource_name' => $this->resource_name,
+            'resource_uuid' => $this->resource->uuid,
             'url' => $this->resource_url,
-            'project' => data_get($this->resource, 'environment.project.name'),
+            'project' => $this->project_name,
             'environment' => $this->environment_name,
             'fqdn' => $this->fqdn,
         ];

resources/views/components/notification/event-grid.blade.php

@@ -14,7 +14,7 @@
             [
                 'key' => 'statusChange',
                 'label' => 'Resource status changes',
-                'helper' => 'Notify when a resource stops or Coolify automatically restarts it.',
+                'helper' => 'Notify when a resource stops unexpectedly (for example a crash) or Coolify automatically restarts it. Stops from Coolify send no notification.',
             ],
             [
                 'key' => 'restartLimitReached',

resources/views/emails/application-status-changes.blade.php

@@ -1,7 +1,7 @@
 <x-emails.layout>
-{{ $name }} has been stopped.
+{{ $name }} has stopped.
 
-If it was your intention to stop this application, you can ignore this email.
+If it was your intention to stop this resource, you can ignore this email.
 
-If not, [check what is going on]({{ $application_url }}).
+If not, [check what is going on]({{ $resource_url }}).
 </x-emails.layout>

tests/Feature/PushServerUpdateJobLastOnlineTest.php

@@ -1,6 +1,7 @@
 <?php
 
 use App\Events\ServiceChecked;
+use App\Jobs\NotifyResourceStoppedJob;
 use App\Jobs\PushServerUpdateJob;
 use App\Models\Environment;
 use App\Models\Project;
@@ -10,6 +11,7 @@ use App\Models\StandalonePostgresql;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Queue;
 
 uses(RefreshDatabase::class);
 
@@ -103,6 +105,7 @@ test('database is not marked exited when an incomplete containers snapshot is em
 });
 
 test('database is marked exited when a complete containers snapshot is empty', function () {
+    Queue::fake([NotifyResourceStoppedJob::class]);
     $team = Team::factory()->create();
     $database = createPushUpdatePostgresql($team, [
         'status' => 'running:healthy',

tests/Feature/ResourceStoppedNotificationTest.php

@@ -0,0 +1,197 @@
+<?php
+
+use App\Actions\Docker\GetContainersStatus;
+use App\Enums\ApplicationDeploymentStatus;
+use App\Events\ServiceChecked;
+use App\Jobs\NotifyResourceStoppedJob;
+use App\Jobs\PushServerUpdateJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Notifications\Application\StatusChanged;
+use Carbon\Carbon;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    InstanceSettings::forceCreate(['id' => 0, 'fqdn' => 'https://coolify.test']);
+    Server::flushIdentityMap();
+    Cache::flush();
+    Event::fake([ServiceChecked::class]);
+    Queue::fake([NotifyResourceStoppedJob::class]);
+    Notification::fake();
+    Carbon::setTestNow(Carbon::parse('2026-11-03 12:00:00', 'UTC'));
+
+    $this->team = Team::factory()->create();
+    $this->team->emailNotificationSettings()->update([
+        'use_instance_email_settings' => true,
+        'status_change_email_notifications' => true,
+    ]);
+    $this->server = Server::factory()->create(['team_id' => $this->team->id]);
+    $this->server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);
+    $this->server->load('settings');
+    $this->destination = StandaloneDocker::where('server_id', $this->server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $this->server->id]);
+    $project = Project::factory()->create(['team_id' => $this->team->id]);
+    $this->environment = Environment::factory()->create(['project_id' => $project->id]);
+});
+
+afterEach(function () {
+    Carbon::setTestNow();
+});
+
+function stoppedNotificationDatabase(array $attributes = []): StandalonePostgresql
+{
+    return StandalonePostgresql::create(array_merge([
+        'uuid' => (string) str()->uuid(),
+        'name' => 'postgres-'.str()->random(8),
+        'postgres_password' => 'secret',
+        'status' => 'running:healthy',
+        'destination_id' => test()->destination->id,
+        'destination_type' => test()->destination->getMorphClass(),
+        'environment_id' => test()->environment->id,
+    ], $attributes));
+}
+
+function stoppedNotificationApplication(): Application
+{
+    return Application::factory()->create([
+        'status' => 'exited',
+        'environment_id' => test()->environment->id,
+        'destination_id' => test()->destination->id,
+        'destination_type' => test()->destination->getMorphClass(),
+    ]);
+}
+
+/**
+ * The Sentinel push of a killed database container: the container still exists, but it exited.
+ */
+function killedDatabasePush(StandalonePostgresql $database): array
+{
+    return [
+        'containers' => [
+            [
+                'name' => $database->uuid,
+                'state' => 'exited',
+                'health_status' => null,
+                'labels' => [
+                    'coolify.managed' => 'true',
+                    'coolify.type' => 'database',
+                    'com.docker.compose.service' => $database->uuid,
+                ],
+            ],
+            [
+                'name' => 'coolify-proxy',
+                'state' => 'running',
+                'labels' => ['coolify.managed' => 'true'],
+            ],
+        ],
+    ];
+}
+
+it('queues a stopped notification when Sentinel reports a killed database', function () {
+    $database = stoppedNotificationDatabase();
+
+    (new PushServerUpdateJob($this->server, killedDatabasePush($database)))->handle();
+
+    expect($database->refresh()->status)->toStartWith('exited');
+    Queue::assertPushed(NotifyResourceStoppedJob::class, fn (NotifyResourceStoppedJob $job): bool => $job->resource->is($database));
+});
+
+it('queues a stopped notification when the SSH status check finds a killed database', function () {
+    $database = stoppedNotificationDatabase();
+    $container = [
+        'Name' => '/'.$database->uuid,
+        'State' => ['Status' => 'exited'],
+        'Config' => ['Labels' => [
+            'coolify.managed' => 'true',
+            'coolify.type' => 'database',
+            'com.docker.compose.service' => $database->uuid,
+        ]],
+    ];
+
+    GetContainersStatus::run($this->server, collect([$container]), collect());
+
+    expect($database->refresh()->status)->toStartWith('exited');
+    Queue::assertPushed(NotifyResourceStoppedJob::class, fn (NotifyResourceStoppedJob $job): bool => $job->resource->is($database));
+});
+
+it('does not queue a notification for a resource that was already stopped', function () {
+    $database = stoppedNotificationDatabase(['status' => 'exited']);
+
+    (new PushServerUpdateJob($this->server, killedDatabasePush($database)))->handle();
+
+    Queue::assertNotPushed(NotifyResourceStoppedJob::class);
+});
+
+it('does not queue a notification when Coolify stops the resource', function () {
+    $database = stoppedNotificationDatabase();
+    NotifyResourceStoppedJob::expectStop($database);
+
+    (new PushServerUpdateJob($this->server, killedDatabasePush($database)))->handle();
+
+    Queue::assertNotPushed(NotifyResourceStoppedJob::class);
+});
+
+it('notifies the team once when the resource is still stopped after the delay', function () {
+    $database = stoppedNotificationDatabase(['status' => 'exited']);
+
+    (new NotifyResourceStoppedJob($database))->handle();
+    (new NotifyResourceStoppedJob($database))->handle();
+
+    Notification::assertSentToTimes($this->team, StatusChanged::class, 1);
+    Notification::assertSentTo($this->team, StatusChanged::class, function (StatusChanged $notification) use ($database): bool {
+        return $notification->resource->is($database)
+            && str($notification->resource_url)->endsWith("/database/{$database->uuid}");
+    });
+
+    Carbon::setTestNow(now()->addMinutes(16));
+    (new NotifyResourceStoppedJob($database))->handle();
+
+    Notification::assertSentToTimes($this->team, StatusChanged::class, 2);
+});
+
+it('does not notify when the resource is running again after the delay', function () {
+    $database = stoppedNotificationDatabase(['status' => 'running:healthy']);
+
+    (new NotifyResourceStoppedJob($database))->handle();
+
+    Notification::assertNothingSent();
+});
+
+it('does not notify when the restart limit stopped the resource', function () {
+    $application = stoppedNotificationApplication();
+    $application->forceFill(['restart_limit_reached' => true])->saveQuietly();
+
+    (new NotifyResourceStoppedJob($application))->handle();
+
+    Notification::assertNothingSent();
+});
+
+it('does not notify while a deployment replaces the application container', function () {
+    $application = stoppedNotificationApplication();
+    $deployment = ApplicationDeploymentQueue::create([
+        'application_id' => (string) $application->id,
+        'deployment_uuid' => (string) str()->uuid(),
+        'pull_request_id' => 0,
+        'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
+    ]);
+
+    (new NotifyResourceStoppedJob($application))->handle();
+    Notification::assertNothingSent();
+
+    $deployment->update(['status' => ApplicationDeploymentStatus::FAILED->value]);
+    (new NotifyResourceStoppedJob($application))->handle();
+    Notification::assertSentTo($this->team, StatusChanged::class);
+});