From f21acdc3e4eb5bb1818a29f113d5acf63c647f77 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:18:50 +0100 Subject: [PATCH] fix(notifications): notify when a resource stops unexpectedly Status checks now queue a delayed check for each application, preview, service part, or database that goes from up to exited. After 2 minutes it sends the "Resource status changes" notification if the resource is still stopped, no deployment runs, and Coolify did not stop it. Each resource sends at most one notification in 15 minutes. Closes #9493 --- CHANGELOG.md | 1 + app/Actions/Application/StopApplication.php | 2 + .../Application/StopApplicationPreview.php | 2 + app/Actions/Database/StopDatabase.php | 2 + app/Actions/Docker/GetContainersStatus.php | 8 + .../Service/DeployServiceApplication.php | 2 + .../Service/RestartServiceApplication.php | 2 + app/Actions/Service/StartService.php | 4 + app/Actions/Service/StopService.php | 3 + .../Service/StopServiceApplication.php | 2 + app/Jobs/NotifyResourceStoppedJob.php | 136 ++++++++++++ app/Jobs/PushServerUpdateJob.php | 7 + .../Application/StatusChanged.php | 96 ++++++--- .../notification/event-grid.blade.php | 2 +- .../application-status-changes.blade.php | 6 +- .../PushServerUpdateJobLastOnlineTest.php | 3 + .../ResourceStoppedNotificationTest.php | 197 ++++++++++++++++++ 17 files changed, 439 insertions(+), 36 deletions(-) create mode 100644 app/Jobs/NotifyResourceStoppedJob.php create mode 100644 tests/Feature/ResourceStoppedNotificationTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index e577e77183..4ffc713bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- Send the **Resource status changes** notification when an application, preview, service, or database stops unexpectedly, for example after a crash or `docker kill`. Before, Coolify sent this notification only when it restarted the proxy or a database TCP proxy. Coolify waits 2 minutes and sends the notification only when the resource is still stopped, so a container that Docker restarts, or a deployment that replaces it, sends none. A stop, restart, or redeploy from Coolify sends no notification, and the same resource sends at most one notification in 15 minutes ([#9493](https://github.com/coollabsio/coolify/issues/9493)) - Fix the "Network error" when you sign in with email in AppFlowy Web. AppFlowy Web runs on its own domain and sends a `Redirect_to` header to GoTrue on the admin domain, and GoTrue blocked the CORS preflight because it did not allow that header. The AppFlowy service template now sets `GOTRUE_CORS_ALLOWED_HEADERS=redirect_to`. For an existing AppFlowy service, add this variable to the `gotrue` service in the Docker Compose file and restart the service ([#10545](https://github.com/coollabsio/coolify/issues/10545)) - List the commit images of Docker Compose applications on the **Rollback** page and in `GET /api/v1/applications/{uuid}/rollback-images`. Since v4.5.0 the list was always empty, because a Compose build names each service image `_`. The page also shows the running commit when the Compose file has services from other images, for example a database ([#12446](https://github.com/coollabsio/coolify/issues/12446)) - Use PostgreSQL 17 instead of MySQL in the Bugsink service template, as the Bugsink Docker Compose sample does, and pin Bugsink to version 2. Before, the template used `mysql:latest` with binary logging on, so the binary logs filled the disk after some weeks and Bugsink stopped. Only new Bugsink services get PostgreSQL. To stop the growth in an existing service, run `PURGE BINARY LOGS BEFORE NOW();` in MySQL, then add `command: ['--disable-log-bin']` to the `mysql` service and restart it ([#10626](https://github.com/coollabsio/coolify/issues/10626)) diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php index 8f8b4bd4f2..c1bb664569 100644 --- a/app/Actions/Application/StopApplication.php +++ b/app/Actions/Application/StopApplication.php @@ -5,6 +5,7 @@ namespace App\Actions\Application; use App\Actions\Server\CleanupDocker; use App\Events\ApplicationStopFailed; use App\Events\ServiceStatusChanged; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\Application; use Illuminate\Support\Facades\Log; use Lorisleiva\Actions\Concerns\AsAction; @@ -17,6 +18,7 @@ class StopApplication public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true, bool $removeContainers = true): ?string { + NotifyResourceStoppedJob::expectStop($application, $application->settings->stopGracePeriodSeconds()); $containerPresent = ! $removeContainers; $servers = collect([$application->destination->server]); if ($application?->additional_servers?->count() > 0) { diff --git a/app/Actions/Application/StopApplicationPreview.php b/app/Actions/Application/StopApplicationPreview.php index 44a6ee15bf..6beb37f434 100644 --- a/app/Actions/Application/StopApplicationPreview.php +++ b/app/Actions/Application/StopApplicationPreview.php @@ -3,6 +3,7 @@ namespace App\Actions\Application; use App\Events\ServiceStatusChanged; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\ApplicationPreview; use Lorisleiva\Actions\Concerns\AsAction; @@ -14,6 +15,7 @@ class StopApplicationPreview { $application = $preview->application; $server = $application->destination->server; + NotifyResourceStoppedJob::expectStop($preview, $application->settings->stopGracePeriodSeconds()); $containers = getCurrentApplicationContainerStatus($server, $application, $preview->pull_request_id); foreach ($containers->pluck('Names') as $containerName) { diff --git a/app/Actions/Database/StopDatabase.php b/app/Actions/Database/StopDatabase.php index 1d1e5c0855..f4a907e85f 100644 --- a/app/Actions/Database/StopDatabase.php +++ b/app/Actions/Database/StopDatabase.php @@ -4,6 +4,7 @@ namespace App\Actions\Database; use App\Actions\Server\CleanupDocker; use App\Events\ServiceStatusChanged; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\BaseModel; use App\Models\StandaloneClickhouse; use App\Models\StandaloneDragonfly; @@ -33,6 +34,7 @@ class StopDatabase $removeContainer = false; } + NotifyResourceStoppedJob::expectStop($database); $this->stopContainer($database, $database->uuid, 30, $removeContainer); // Reset restart tracking when database is manually stopped diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index c59545a9dd..53d8c2f28b 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -9,6 +9,7 @@ use App\Actions\Database\StopDatabaseProxy; use App\Actions\Service\StopServiceApplication; use App\Actions\Shared\ComplexStatusCheck; use App\Events\ServiceChecked; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\Application; use App\Models\ApplicationPreview; use App\Models\Server; @@ -125,6 +126,11 @@ class GetContainersStatus implements ShouldBeUnique $databases = $this->server->databases(); $services = $this->server->services()->get(); $previews = $this->server->previews(); + $upResourceIds = NotifyResourceStoppedJob::upResourceIds(collect($this->applications) + ->concat($previews) + ->concat($databases) + ->concat($services->flatMap(fn ($service) => $service->applications)) + ->concat($services->flatMap(fn ($service) => $service->databases))); $foundApplications = []; $foundApplicationPreviews = []; $foundDatabases = []; @@ -555,6 +561,8 @@ class GetContainersStatus implements ShouldBeUnique // Aggregate multi-container service statuses $this->aggregateServiceContainerStatuses($services); + NotifyResourceStoppedJob::dispatchForStopped($upResourceIds); + ServiceChecked::dispatch($this->server->team->id); } diff --git a/app/Actions/Service/DeployServiceApplication.php b/app/Actions/Service/DeployServiceApplication.php index c5370eb51f..3893f11d80 100644 --- a/app/Actions/Service/DeployServiceApplication.php +++ b/app/Actions/Service/DeployServiceApplication.php @@ -4,6 +4,7 @@ namespace App\Actions\Service; use App\Actions\Shared\EnsureContentFilesOnServer; use App\Events\ServiceStartFinished; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; use Lorisleiva\Actions\Concerns\AsAction; @@ -19,6 +20,7 @@ class DeployServiceApplication { $service = $serviceApplication->service; $composeServiceName = $serviceApplication->name; + NotifyResourceStoppedJob::expectStop($serviceApplication); $service->parse(); StartService::queueFileStorageSync($service); diff --git a/app/Actions/Service/RestartServiceApplication.php b/app/Actions/Service/RestartServiceApplication.php index 94c06ca653..76b8d82a91 100644 --- a/app/Actions/Service/RestartServiceApplication.php +++ b/app/Actions/Service/RestartServiceApplication.php @@ -2,6 +2,7 @@ namespace App\Actions\Service; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; use Lorisleiva\Actions\Concerns\AsAction; @@ -21,6 +22,7 @@ class RestartServiceApplication $service = $serviceApplication->service; $server = $service->destination->server; $containerName = escapeshellarg($serviceApplication->name.'-'.$service->uuid); + NotifyResourceStoppedJob::expectStop($serviceApplication); instant_remote_process([ "docker restart {$containerName}", diff --git a/app/Actions/Service/StartService.php b/app/Actions/Service/StartService.php index fa4d517487..e0d80d759b 100644 --- a/app/Actions/Service/StartService.php +++ b/app/Actions/Service/StartService.php @@ -4,6 +4,7 @@ namespace App\Actions\Service; use App\Actions\Shared\EnsureContentFilesOnServer; use App\Events\ServiceStartFinished; +use App\Jobs\NotifyResourceStoppedJob; use App\Jobs\ServerFilesFromServerJob; use App\Models\LocalFileVolume; use App\Models\Service; @@ -30,6 +31,9 @@ class StartService // Fetch remote secrets before stopping: if the secret manager fails, the service keeps running. // saveComposeConfigs() below reuses the fetched secrets. $service->ensureRemoteSecretsResolvable($service->environment_variables()->get()); + // Compose recreates the containers, so a status check can find them stopped for a moment. + $service->applications()->get()->concat($service->databases()->get()) + ->each(fn (ServiceApplication|ServiceDatabase $part) => NotifyResourceStoppedJob::expectStop($part)); if ($this->shouldStopBeforeStarting($pullLatestImages, $stopBeforeStart)) { StopService::run(service: $service, dockerCleanup: false); } diff --git a/app/Actions/Service/StopService.php b/app/Actions/Service/StopService.php index ce6d7e25ff..155e39f706 100644 --- a/app/Actions/Service/StopService.php +++ b/app/Actions/Service/StopService.php @@ -5,6 +5,7 @@ namespace App\Actions\Service; use App\Actions\Server\CleanupDocker; use App\Enums\ProcessStatus; use App\Events\ServiceStatusChanged; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\Server; use App\Models\Service; use Lorisleiva\Actions\Concerns\AsAction; @@ -39,10 +40,12 @@ class StopService $containersToStop = []; $applications = $service->applications()->get(); foreach ($applications as $application) { + NotifyResourceStoppedJob::expectStop($application); $containersToStop[] = "{$application->name}-{$service->uuid}"; } $dbs = $service->databases()->get(); foreach ($dbs as $db) { + NotifyResourceStoppedJob::expectStop($db); $containersToStop[] = "{$db->name}-{$service->uuid}"; } diff --git a/app/Actions/Service/StopServiceApplication.php b/app/Actions/Service/StopServiceApplication.php index 1b53472656..47c8568dc1 100644 --- a/app/Actions/Service/StopServiceApplication.php +++ b/app/Actions/Service/StopServiceApplication.php @@ -3,6 +3,7 @@ namespace App\Actions\Service; use App\Events\ServiceStatusChanged; +use App\Jobs\NotifyResourceStoppedJob; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; use Lorisleiva\Actions\Concerns\AsAction; @@ -18,6 +19,7 @@ class StopServiceApplication $service = $serviceApplication->service; $server = $service->destination->server; $containerName = escapeshellarg($serviceApplication->name.'-'.$service->uuid); + NotifyResourceStoppedJob::expectStop($serviceApplication); if ($removeContainer) { $commands = ["docker rm -f {$containerName}"]; diff --git a/app/Jobs/NotifyResourceStoppedJob.php b/app/Jobs/NotifyResourceStoppedJob.php new file mode 100644 index 0000000000..f05362da37 --- /dev/null +++ b/app/Jobs/NotifyResourceStoppedJob.php @@ -0,0 +1,136 @@ +onQueue('high'); + $this->delay(now()->addSeconds(self::CONFIRMATION_DELAY_SECONDS)); + } + + /** + * Call before Coolify stops or recreates the containers of a resource, so that the stop sends no notification. + */ + public static function expectStop(BaseModel $resource, int $stopTimeoutSeconds = 30): void + { + Cache::put(self::expectedStopKey($resource), true, now()->addSeconds($stopTimeoutSeconds + self::EXPECTED_STOP_SECONDS)); + } + + /** + * Read which resources are up, before a status check changes their status. + * + * @param iterable $resources + * @return array, list> + */ + public static function upResourceIds(iterable $resources): array + { + $ids = []; + foreach ($resources as $resource) { + if (filled($resource->status) && ! self::isExited($resource->status)) { + $ids[$resource::class][] = $resource->getKey(); + } + } + + return $ids; + } + + /** + * After a status check: queue a notification for each resource that was up before the check and is exited now. + * + * @param array, list> $upResourceIds from upResourceIds() + */ + public static function dispatchForStopped(array $upResourceIds): void + { + foreach ($upResourceIds as $class => $ids) { + $class::query() + ->whereKey($ids) + ->where('status', 'like', 'exited%') + ->get() + ->reject(fn (BaseModel $resource): bool => self::stopIsExpected($resource)) + ->each(fn (BaseModel $resource) => self::dispatch($resource)); + } + } + + public function handle(): void + { + $resource = $this->resource::query()->whereKey($this->resource->getKey())->first(); + if ($resource === null + || ! self::isExited($resource->status) + || (bool) $resource->restart_limit_reached + || self::stopIsExpected($resource) + || self::isDeploying($resource)) { + return; + } + + $team = $resource instanceof ApplicationPreview ? $resource->application?->team() : $resource->team(); + $team?->notify(new StatusChanged($resource)); + } + + private static function isExited(?string $status): bool + { + return str($status)->startsWith('exited'); + } + + private static function stopIsExpected(BaseModel $resource): bool + { + return Cache::has(self::expectedStopKey($resource)); + } + + private static function expectedStopKey(BaseModel $resource): string + { + return 'resource-stop-expected:'.$resource->getMorphClass().':'.$resource->getKey(); + } + + private static function isDeploying(BaseModel $resource): bool + { + [$applicationId, $pullRequestId] = match (true) { + $resource instanceof Application => [$resource->id, 0], + $resource instanceof ApplicationPreview => [$resource->application_id, $resource->pull_request_id], + default => [null, null], + }; + if ($applicationId === null) { + return false; + } + + return ApplicationDeploymentQueue::query() + ->where('application_id', (string) $applicationId) + ->where('pull_request_id', $pullRequestId) + ->whereIn('status', [ApplicationDeploymentStatus::QUEUED->value, ApplicationDeploymentStatus::IN_PROGRESS->value]) + ->exists(); + } +} diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index ea15d0a5f6..0e9c074f25 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -247,6 +247,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid'); $this->allServiceApplicationIds = $this->serviceApplicationsById->keys(); $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys(); + $upResourceIds = NotifyResourceStoppedJob::upResourceIds($this->applications->whereIn('id', $this->allApplicationIds) + ->concat($this->previews) + ->concat($this->databases) + ->concat($this->serviceApplicationsById) + ->concat($this->serviceDatabasesById)); // Owners of application containers outside this server's applications, in one query. $foreignApplicationIdsByUuid = containerApplicationIdsByUuid( @@ -405,6 +410,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced // Aggregate multi-container service statuses $this->aggregateServiceContainerStatuses(); + NotifyResourceStoppedJob::dispatchForStopped($upResourceIds); + $this->checkLogDrainContainer(); ServiceChecked::dispatch($this->server->team_id); diff --git a/app/Notifications/Application/StatusChanged.php b/app/Notifications/Application/StatusChanged.php index ef61b7e6a5..4f04616811 100644 --- a/app/Notifications/Application/StatusChanged.php +++ b/app/Notifications/Application/StatusChanged.php @@ -2,39 +2,62 @@ namespace App\Notifications\Application; +use App\Contracts\ThrottledNotification; use App\Models\Application; +use App\Models\ApplicationPreview; +use App\Models\BaseModel; +use App\Models\ServiceApplication; +use App\Models\ServiceDatabase; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; use App\Notifications\Dto\SlackMessage; +use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Messages\MailMessage; -class StatusChanged extends CustomEmailNotification +/** + * A resource (application, preview, service part, or database) stopped without a stop from Coolify. + */ +class StatusChanged extends CustomEmailNotification implements ThrottledNotification { public string $resource_name; - public string $project_uuid; + public ?string $project_name; - public string $environment_uuid; + public ?string $project_uuid; - public string $environment_name; + public ?string $environment_uuid; + + public ?string $environment_name; public ?string $resource_url = null; public ?string $fqdn; - public function __construct(public Application $resource) + public function __construct(public BaseModel $resource) { $this->onQueue('high'); - $this->resource_name = data_get($resource, 'name'); - $this->project_uuid = data_get($resource, 'environment.project.uuid'); - $this->environment_uuid = data_get($resource, 'environment.uuid'); - $this->environment_name = data_get($resource, 'environment.name'); + $environment = data_get($resource, 'environment') + ?? data_get($resource, 'application.environment') + ?? data_get($resource, 'service.environment'); + $this->resource_name = $resource instanceof ApplicationPreview + ? data_get($resource, 'application.name').' PR #'.$resource->pull_request_id + : (string) data_get($resource, 'name'); + $this->project_name = data_get($environment, 'project.name'); + $this->project_uuid = data_get($environment, 'project.uuid'); + $this->environment_uuid = data_get($environment, 'uuid'); + $this->environment_name = data_get($environment, 'name'); $this->fqdn = data_get($resource, 'fqdn', null); if (str($this->fqdn)->explode(',')->count() > 1) { $this->fqdn = str($this->fqdn)->explode(',')->first(); } - $this->resource_url = base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/application/{$this->resource->uuid}"; + [$type, $uuid] = match (true) { + $resource instanceof Application => ['application', $resource->uuid], + $resource instanceof ApplicationPreview => ['application', data_get($resource, 'application.uuid')], + $resource instanceof ServiceApplication, $resource instanceof ServiceDatabase => ['service', data_get($resource, 'service.uuid')], + default => ['database', $resource->uuid], + }; + $this->resource_url = base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/{$type}/{$uuid}"; } public function via(object $notifiable): array @@ -42,14 +65,26 @@ class StatusChanged extends CustomEmailNotification return $notifiable->getEnabledChannels('status_change'); } + public function throttleSubject(): ?Model + { + return $this->resource; + } + + /** + * A resource that stops again soon after it came back sends no second notification. + */ + public function throttleIntervalMinutes(): int + { + return 15; + } + public function toMail(): MailMessage { $mail = new MailMessage; - $fqdn = $this->fqdn; - $mail->subject("Coolify: {$this->resource_name} has been stopped"); + $mail->subject("Coolify: {$this->resource_name} has stopped"); $mail->view('emails.application-status-changes', [ 'name' => $this->resource_name, - 'fqdn' => $fqdn, + 'fqdn' => $this->fqdn, 'resource_url' => $this->resource_url, ]); @@ -59,8 +94,8 @@ class StatusChanged extends CustomEmailNotification public function toDiscord(): DiscordMessage { return new DiscordMessage( - title: ':cross_mark: Application stopped', - description: '[Open Application in Coolify]('.$this->resource_url.')', + title: ':cross_mark: Resource stopped', + description: "{$this->resource_name} has stopped.\n\n[Open Resource in Coolify]({$this->resource_url})", color: DiscordMessage::errorColor(), isCritical: true, ); @@ -68,13 +103,11 @@ class StatusChanged extends CustomEmailNotification public function toTelegram(): array { - $message = 'Coolify: '.$this->resource_name.' has been stopped.'; - return [ - 'message' => $message, + 'message' => "Coolify: {$this->resource_name} has stopped.", 'buttons' => [ [ - 'text' => 'Open Application in Coolify', + 'text' => 'Open Resource in Coolify', 'url' => $this->resource_url, ], ], @@ -83,15 +116,13 @@ class StatusChanged extends CustomEmailNotification public function toPushover(): PushoverMessage { - $message = $this->resource_name.' has been stopped.'; - return new PushoverMessage( - title: 'Application stopped', + title: 'Resource stopped', level: 'error', - message: $message, + message: "{$this->resource_name} has stopped.", buttons: [ [ - 'text' => 'Open Application in Coolify', + 'text' => 'Open Resource in Coolify', 'url' => $this->resource_url, ], ], @@ -100,15 +131,13 @@ class StatusChanged extends CustomEmailNotification public function toSlack(): SlackMessage { - $title = 'Application stopped'; - $description = "{$this->resource_name} has been stopped"; - - $description .= "\n\n*Project:* ".data_get($this->resource, 'environment.project.name'); + $description = "{$this->resource_name} has stopped"; + $description .= "\n\n*Project:* {$this->project_name}"; $description .= "\n*Environment:* {$this->environment_name}"; - $description .= "\n*Application URL:* {$this->resource_url}"; + $description .= "\n*Resource URL:* {$this->resource_url}"; return new SlackMessage( - title: $title, + title: 'Resource stopped', description: $description, color: SlackMessage::errorColor() ); @@ -118,12 +147,15 @@ class StatusChanged extends CustomEmailNotification { return [ 'success' => false, - 'message' => 'Application stopped', + 'message' => 'Resource stopped', 'event' => 'status_changed', + 'status' => 'exited', 'application_name' => $this->resource_name, 'application_uuid' => $this->resource->uuid, + 'resource_name' => $this->resource_name, + 'resource_uuid' => $this->resource->uuid, 'url' => $this->resource_url, - 'project' => data_get($this->resource, 'environment.project.name'), + 'project' => $this->project_name, 'environment' => $this->environment_name, 'fqdn' => $this->fqdn, ]; diff --git a/resources/views/components/notification/event-grid.blade.php b/resources/views/components/notification/event-grid.blade.php index 55b452cf5e..65c4a6e262 100644 --- a/resources/views/components/notification/event-grid.blade.php +++ b/resources/views/components/notification/event-grid.blade.php @@ -14,7 +14,7 @@ [ 'key' => 'statusChange', 'label' => 'Resource status changes', - 'helper' => 'Notify when a resource stops or Coolify automatically restarts it.', + 'helper' => 'Notify when a resource stops unexpectedly (for example a crash) or Coolify automatically restarts it. Stops from Coolify send no notification.', ], [ 'key' => 'restartLimitReached', diff --git a/resources/views/emails/application-status-changes.blade.php b/resources/views/emails/application-status-changes.blade.php index 30966060f5..eab65ba44e 100644 --- a/resources/views/emails/application-status-changes.blade.php +++ b/resources/views/emails/application-status-changes.blade.php @@ -1,7 +1,7 @@ -{{ $name }} has been stopped. +{{ $name }} has stopped. -If it was your intention to stop this application, you can ignore this email. +If it was your intention to stop this resource, you can ignore this email. -If not, [check what is going on]({{ $application_url }}). +If not, [check what is going on]({{ $resource_url }}). diff --git a/tests/Feature/PushServerUpdateJobLastOnlineTest.php b/tests/Feature/PushServerUpdateJobLastOnlineTest.php index 28e1ccead4..58b0df25e0 100644 --- a/tests/Feature/PushServerUpdateJobLastOnlineTest.php +++ b/tests/Feature/PushServerUpdateJobLastOnlineTest.php @@ -1,6 +1,7 @@ create(); $database = createPushUpdatePostgresql($team, [ 'status' => 'running:healthy', diff --git a/tests/Feature/ResourceStoppedNotificationTest.php b/tests/Feature/ResourceStoppedNotificationTest.php new file mode 100644 index 0000000000..263852e1a8 --- /dev/null +++ b/tests/Feature/ResourceStoppedNotificationTest.php @@ -0,0 +1,197 @@ + 0, 'fqdn' => 'https://coolify.test']); + Server::flushIdentityMap(); + Cache::flush(); + Event::fake([ServiceChecked::class]); + Queue::fake([NotifyResourceStoppedJob::class]); + Notification::fake(); + Carbon::setTestNow(Carbon::parse('2026-11-03 12:00:00', 'UTC')); + + $this->team = Team::factory()->create(); + $this->team->emailNotificationSettings()->update([ + 'use_instance_email_settings' => true, + 'status_change_email_notifications' => true, + ]); + $this->server = Server::factory()->create(['team_id' => $this->team->id]); + $this->server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]); + $this->server->load('settings'); + $this->destination = StandaloneDocker::where('server_id', $this->server->id)->first() + ?? StandaloneDocker::factory()->create(['server_id' => $this->server->id]); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $project->id]); +}); + +afterEach(function () { + Carbon::setTestNow(); +}); + +function stoppedNotificationDatabase(array $attributes = []): StandalonePostgresql +{ + return StandalonePostgresql::create(array_merge([ + 'uuid' => (string) str()->uuid(), + 'name' => 'postgres-'.str()->random(8), + 'postgres_password' => 'secret', + 'status' => 'running:healthy', + 'destination_id' => test()->destination->id, + 'destination_type' => test()->destination->getMorphClass(), + 'environment_id' => test()->environment->id, + ], $attributes)); +} + +function stoppedNotificationApplication(): Application +{ + return Application::factory()->create([ + 'status' => 'exited', + 'environment_id' => test()->environment->id, + 'destination_id' => test()->destination->id, + 'destination_type' => test()->destination->getMorphClass(), + ]); +} + +/** + * The Sentinel push of a killed database container: the container still exists, but it exited. + */ +function killedDatabasePush(StandalonePostgresql $database): array +{ + return [ + 'containers' => [ + [ + 'name' => $database->uuid, + 'state' => 'exited', + 'health_status' => null, + 'labels' => [ + 'coolify.managed' => 'true', + 'coolify.type' => 'database', + 'com.docker.compose.service' => $database->uuid, + ], + ], + [ + 'name' => 'coolify-proxy', + 'state' => 'running', + 'labels' => ['coolify.managed' => 'true'], + ], + ], + ]; +} + +it('queues a stopped notification when Sentinel reports a killed database', function () { + $database = stoppedNotificationDatabase(); + + (new PushServerUpdateJob($this->server, killedDatabasePush($database)))->handle(); + + expect($database->refresh()->status)->toStartWith('exited'); + Queue::assertPushed(NotifyResourceStoppedJob::class, fn (NotifyResourceStoppedJob $job): bool => $job->resource->is($database)); +}); + +it('queues a stopped notification when the SSH status check finds a killed database', function () { + $database = stoppedNotificationDatabase(); + $container = [ + 'Name' => '/'.$database->uuid, + 'State' => ['Status' => 'exited'], + 'Config' => ['Labels' => [ + 'coolify.managed' => 'true', + 'coolify.type' => 'database', + 'com.docker.compose.service' => $database->uuid, + ]], + ]; + + GetContainersStatus::run($this->server, collect([$container]), collect()); + + expect($database->refresh()->status)->toStartWith('exited'); + Queue::assertPushed(NotifyResourceStoppedJob::class, fn (NotifyResourceStoppedJob $job): bool => $job->resource->is($database)); +}); + +it('does not queue a notification for a resource that was already stopped', function () { + $database = stoppedNotificationDatabase(['status' => 'exited']); + + (new PushServerUpdateJob($this->server, killedDatabasePush($database)))->handle(); + + Queue::assertNotPushed(NotifyResourceStoppedJob::class); +}); + +it('does not queue a notification when Coolify stops the resource', function () { + $database = stoppedNotificationDatabase(); + NotifyResourceStoppedJob::expectStop($database); + + (new PushServerUpdateJob($this->server, killedDatabasePush($database)))->handle(); + + Queue::assertNotPushed(NotifyResourceStoppedJob::class); +}); + +it('notifies the team once when the resource is still stopped after the delay', function () { + $database = stoppedNotificationDatabase(['status' => 'exited']); + + (new NotifyResourceStoppedJob($database))->handle(); + (new NotifyResourceStoppedJob($database))->handle(); + + Notification::assertSentToTimes($this->team, StatusChanged::class, 1); + Notification::assertSentTo($this->team, StatusChanged::class, function (StatusChanged $notification) use ($database): bool { + return $notification->resource->is($database) + && str($notification->resource_url)->endsWith("/database/{$database->uuid}"); + }); + + Carbon::setTestNow(now()->addMinutes(16)); + (new NotifyResourceStoppedJob($database))->handle(); + + Notification::assertSentToTimes($this->team, StatusChanged::class, 2); +}); + +it('does not notify when the resource is running again after the delay', function () { + $database = stoppedNotificationDatabase(['status' => 'running:healthy']); + + (new NotifyResourceStoppedJob($database))->handle(); + + Notification::assertNothingSent(); +}); + +it('does not notify when the restart limit stopped the resource', function () { + $application = stoppedNotificationApplication(); + $application->forceFill(['restart_limit_reached' => true])->saveQuietly(); + + (new NotifyResourceStoppedJob($application))->handle(); + + Notification::assertNothingSent(); +}); + +it('does not notify while a deployment replaces the application container', function () { + $application = stoppedNotificationApplication(); + $deployment = ApplicationDeploymentQueue::create([ + 'application_id' => (string) $application->id, + 'deployment_uuid' => (string) str()->uuid(), + 'pull_request_id' => 0, + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + ]); + + (new NotifyResourceStoppedJob($application))->handle(); + Notification::assertNothingSent(); + + $deployment->update(['status' => ApplicationDeploymentStatus::FAILED->value]); + (new NotifyResourceStoppedJob($application))->handle(); + Notification::assertSentTo($this->team, StatusChanged::class); +});