fix(notifications): throttle repeated notifications per resource

High disk usage notifications were sent every ~10 minutes on Coolify Cloud
because the cache rate limiter did not hold across Sentinel-triggered checks.

- Add a notification_throttles table and NotificationThrottle helper that
  atomically records when a notification was sent for a resource.
- Add the ThrottledNotification contract; Team::notify() skips a throttled
  notification that was already sent within its interval.
- High disk usage: configurable per-server interval
  (server_disk_usage_notification_interval_hours, default 24h) in
  Server > Advanced and the servers API.
- Container restarted (coolify-proxy, database TCP proxy): at most one
  notification per hour per server/database during a restart loop.
- Move the unreachable and missing-backup markers into the new table and
  migrate existing data (servers.unreachable_notification_sent and
  scheduled_database_backups.missing_backup_notification_sent_at).
- Regenerate OpenAPI files.
This commit is contained in:
Andras Bacsai committed 2026-09-29 11:29:39 +02:00
1 parent 49086c82a8
commit 99c840922c
33 files changed
+786 -81

No files matched your search

+2
View File
@@ -5,6 +5,8 @@
- When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround.
- Test old reports against the current branch because later changes can make the report obsolete.
- Use the same regression test before and after the production change so the result shows the behavior difference.
- Several dev instances can run from one checkout (see `./scripts/dev urls`). After you add a migration, run it on each running instance of that checkout, or those instances fail on the new code.
- The checkout is shared with other sessions. When the full suite fails, rerun each failing file alone and compare with a clean `git archive HEAD` copy before you connect a failure to your change.
- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.
- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.
@@ -3,6 +3,7 @@
namespace App\Console\Commands;
use App\Models\Server;
use App\Notifications\Server\Unreachable;
use Illuminate\Console\Command;
class CleanupUnreachableServers extends Command
@@ -14,7 +15,7 @@ class CleanupUnreachableServers extends Command
public function handle()
{
echo "Running unreachable server cleanup...\n";
$servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
$servers = Server::where('unreachable_count', '>=', 3)->whereHas('notificationThrottles', fn ($query) => $query->where('notification', Unreachable::class))->where('updated_at', '<', now()->subDays(7))->get();
if ($servers->count() > 0) {
foreach ($servers as $server) {
echo "Cleanup unreachable server ($server->id) with name $server->name";
+19
View File
@@ -0,0 +1,19 @@
<?php
namespace App\Contracts;
use Illuminate\Database\Eloquent\Model;
/**
* A notification that is sent at most once per interval for the same resource.
* Team::notify() checks the throttle before it sends the notification.
*/
interface ThrottledNotification
{
/**
* The resource the throttle belongs to. Return null to send without a throttle.
*/
public function throttleSubject(): ?Model;
public function throttleIntervalMinutes(): int;
}
@@ -5119,7 +5119,6 @@ class DatabasesController extends Controller
'created_at',
'updated_at',
'last_execution_at',
'missing_backup_notification_sent_at',
])->fill([
'uuid' => new_public_id(),
'database_id' => $newDatabase->id,
@@ -187,6 +187,7 @@ class ServersController extends Controller
$server->load(['settings']);
}
$server->append('unreachable_notification_sent');
$settings = $this->removeSensitiveDataFromSettings($server->settings);
$server = $this->removeSensitiveData($server);
data_set($server, 'settings', $settings);
@@ -645,6 +646,7 @@ class ServersController extends Controller
'deployment_queue_limit' => ['type' => 'integer', 'description' => 'Maximum number of queued deployments.'],
'server_disk_usage_notification_threshold' => ['type' => 'integer', 'description' => 'Server disk usage notification threshold (%).'],
'server_disk_usage_check_frequency' => ['type' => 'string', 'description' => 'Cron expression for disk usage check frequency.'],
'server_disk_usage_notification_interval_hours' => ['type' => 'integer', 'description' => 'Minimum hours between high disk usage notifications (1-720). Default: 24.'],
'connection_timeout' => ['type' => 'integer', 'description' => 'SSH connection timeout in seconds (1-300). Default: 10.'],
],
),
@@ -682,7 +684,7 @@ class ServersController extends Controller
)]
public function update_server(Request $request)
{
$allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout', 'is_terminal_enabled'];
$allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout', 'is_terminal_enabled'];
$teamId = getTeamIdFromToken();
if (is_null($teamId)) {
@@ -708,6 +710,7 @@ class ServersController extends Controller
'deployment_queue_limit' => 'integer|min:1',
'server_disk_usage_notification_threshold' => 'integer|min:1|max:100',
'server_disk_usage_check_frequency' => 'string',
'server_disk_usage_notification_interval_hours' => 'integer|min:1|max:720',
'connection_timeout' => 'integer|min:1|max:300',
'is_terminal_enabled' => 'boolean|nullable',
], [
@@ -790,7 +793,7 @@ class ServersController extends Controller
]);
}
$advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout']);
$advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout']);
if (! empty($advancedSettings)) {
$server->settings()->update(array_filter($advancedSettings, fn ($value) => ! is_null($value)));
}
+6 -5
View File
@@ -2,6 +2,7 @@
namespace App\Jobs;
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Notifications\Database\BackupMissing;
use Illuminate\Bus\Queueable;
@@ -39,10 +40,6 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu
return;
}
if ($backup->missing_backup_notification_sent_at?->greaterThanOrEqualTo($lastActivityAt)) {
return;
}
if (! $backup->team) {
Log::warning("Cannot send missing backup notification for backup {$backup->id}: team not found");
@@ -53,7 +50,11 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu
return;
}
// Send once for each period without backup activity.
if (! NotificationThrottle::claim($backup, BackupMissing::class, $lastActivityAt)) {
return;
}
$backup->team->notify(new BackupMissing($backup, $lastExecutionAt));
$backup->forceFill(['missing_backup_notification_sent_at' => now()])->save();
}
}
+2 -2
View File
@@ -799,7 +799,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
try {
if (CheckProxy::run($this->server)) {
StartProxy::run($this->server, async: false);
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
}
} catch (\Throwable $e) {
}
@@ -841,7 +841,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
})->first();
if (! $tcpProxyContainerFound) {
StartDatabaseProxy::dispatch($database);
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database));
}
} elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
// Clean up orphaned proxy containers when is_public=false
+1 -1
View File
@@ -85,7 +85,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
$shouldStart = CheckProxy::run($this->server);
if ($shouldStart) {
StartProxy::run($this->server, async: false);
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
}
} catch (\Throwable $e) {
}
+3 -16
View File
@@ -10,8 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\TimeoutExceededException;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\RateLimiter;
use Laravel\Horizon\Contracts\Silenced;
class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -31,7 +31,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public function failed(?\Throwable $exception): void
{
if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
if ($exception instanceof TimeoutExceededException) {
Log::warning('ServerStorageCheckJob timed out', [
'server_id' => $this->server->id,
'server_name' => $this->server->name,
@@ -58,20 +58,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
return 'No percentage could be retrieved.';
}
if ($this->percentage > $serverDiskUsageNotificationThreshold) {
$executed = RateLimiter::attempt(
'high-disk-usage:'.$this->server->id,
$maxAttempts = 0,
function () use ($team, $serverDiskUsageNotificationThreshold) {
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
},
$decaySeconds = 3600,
);
if (! $executed) {
return 'Too many messages sent!';
}
} else {
RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
}
} catch (\Throwable $e) {
return handleError($e);
+5
View File
@@ -18,6 +18,9 @@ class Advanced extends Component
#[Validate(['required', 'integer', 'min:1', 'max:99'])]
public int|string $serverDiskUsageNotificationThreshold = 50;
#[Validate(['required', 'integer', 'min:1', 'max:720'])]
public int|string $serverDiskUsageNotificationIntervalHours = 24;
#[Validate(['required', 'integer', 'min:1'])]
public int|string $concurrentBuilds = 1;
@@ -52,6 +55,7 @@ class Advanced extends Component
$this->server->settings->backup_compression_cpu_percentage = $this->backupCompressionCpuPercentage;
$this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold;
$this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency;
$this->server->settings->server_disk_usage_notification_interval_hours = $this->serverDiskUsageNotificationIntervalHours;
$this->server->settings->save();
} else {
$this->concurrentBuilds = $this->server->settings->concurrent_builds;
@@ -60,6 +64,7 @@ class Advanced extends Component
$this->backupCompressionCpuPercentage = $this->server->settings->backup_compression_cpu_percentage;
$this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold;
$this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency;
$this->serverDiskUsageNotificationIntervalHours = $this->server->settings->server_disk_usage_notification_interval_hours;
}
}
+111
View File
@@ -0,0 +1,111 @@
<?php
namespace App\Models;
use App\Contracts\ThrottledNotification;
use Carbon\CarbonInterface;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\MorphTo;
/**
* Records when a notification was last sent for a resource, so repeated checks
* (across queue workers) do not send the same notification again.
*/
class NotificationThrottle extends Model
{
public $timestamps = false;
protected $fillable = [
'notifiable_type',
'notifiable_id',
'notification',
'sent_at',
];
protected function casts(): array
{
return [
'sent_at' => 'datetime',
];
}
public function notifiable(): MorphTo
{
return $this->morphTo();
}
/**
* Atomically reserve the right to send a notification.
*
* Returns true when the notification was never sent, or when it was last sent
* before $sentBefore. Without $sentBefore, it is sent only once until released.
*/
public static function claim(Model $notifiable, string $notification, ?CarbonInterface $sentBefore = null): bool
{
if ($sentBefore !== null) {
$renewed = static::query()
->whereMorphedTo('notifiable', $notifiable)
->where('notification', $notification)
->where('sent_at', '<', $sentBefore)
->update(['sent_at' => now()]);
if ($renewed === 1) {
return true;
}
}
return static::query()->insertOrIgnore([
'notifiable_type' => $notifiable->getMorphClass(),
'notifiable_id' => $notifiable->getKey(),
'notification' => $notification,
'sent_at' => now(),
]) === 1;
}
/**
* Claim the throttle a notification declares. Notifications without a throttle subject are always allowed.
*/
public static function claimFor(ThrottledNotification $notification): bool
{
$subject = $notification->throttleSubject();
if ($subject === null) {
return true;
}
return static::claim($subject, $notification::class, now()->subMinutes($notification->throttleIntervalMinutes()));
}
/**
* Mark a notification as sent without sending it.
*/
public static function record(Model $notifiable, string $notification): void
{
static::query()->updateOrInsert(
[
'notifiable_type' => $notifiable->getMorphClass(),
'notifiable_id' => $notifiable->getKey(),
'notification' => $notification,
],
['sent_at' => now()],
);
}
/**
* Forget a sent notification. Returns true when a record was removed.
*/
public static function release(Model $notifiable, string $notification): bool
{
return static::query()
->whereMorphedTo('notifiable', $notifiable)
->where('notification', $notification)
->delete() > 0;
}
public static function wasSent(Model $notifiable, string $notification): bool
{
return static::query()
->whereMorphedTo('notifiable', $notifiable)
->where('notification', $notification)
->exists();
}
}
-1
View File
@@ -18,7 +18,6 @@ class ScheduledDatabaseBackup extends BaseModel
'database_backup_retention_max_storage_locally' => 'float',
'database_backup_retention_max_storage_s3' => 'float',
'missing_backup_notification_days' => 'integer',
'missing_backup_notification_sent_at' => 'datetime',
'last_execution_at' => 'datetime',
'next_run_at' => 'datetime',
];
+21 -9
View File
@@ -29,6 +29,7 @@ use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
@@ -235,6 +236,7 @@ class Server extends BaseModel
});
$server->settings()->delete();
$server->sslCertificates()->delete();
$server->notificationThrottles()->delete();
});
static::updated(function () {
@@ -280,7 +282,6 @@ class Server extends BaseModel
'logdrain_newrelic_license_key' => 'encrypted',
'delete_unused_volumes' => 'boolean',
'delete_unused_networks' => 'boolean',
'unreachable_notification_sent' => 'boolean',
'force_disabled' => 'boolean',
'sentinel_waiting_since' => 'datetime',
];
@@ -1526,6 +1527,19 @@ $siteAddress {
return $this->belongsTo(CloudProviderToken::class);
}
public function notificationThrottles(): MorphMany
{
return $this->morphMany(NotificationThrottle::class, 'notifiable');
}
/**
* True while an Unreachable notification was sent and no Reachable notification followed.
*/
protected function unreachableNotificationSent(): Attribute
{
return Attribute::get(fn (): bool => NotificationThrottle::wasSent($this, Unreachable::class));
}
public function sslCertificates()
{
return $this->hasMany(SslCertificate::class);
@@ -1777,18 +1791,16 @@ $siteAddress {
public function sendReachableNotification()
{
$this->unreachable_notification_sent = false;
$this->save();
$this->refresh();
$this->team->notify(new Reachable($this));
if (NotificationThrottle::release($this, Unreachable::class)) {
$this->team->notify(new Reachable($this));
}
}
public function sendUnreachableNotification()
{
$this->unreachable_notification_sent = true;
$this->save();
$this->refresh();
$this->team->notify(new Unreachable($this));
if (NotificationThrottle::claim($this, Unreachable::class)) {
$this->team->notify(new Unreachable($this));
}
}
public function validateConnection(bool $justCheckingNewKey = false)
+2
View File
@@ -117,6 +117,7 @@ class ServerSetting extends Model
'server_disk_usage_notification_threshold',
'is_sentinel_debug_enabled',
'server_disk_usage_check_frequency',
'server_disk_usage_notification_interval_hours',
'is_terminal_enabled',
'deployment_queue_limit',
'backup_compression_cpu_percentage',
@@ -170,6 +171,7 @@ class ServerSetting extends Model
'compose_version_checked_at' => 'datetime',
'backup_compression_cpu_percentage' => 'integer',
'docker_cleanup_next_run_at' => 'datetime',
'server_disk_usage_notification_interval_hours' => 'integer',
];
/**
+19 -2
View File
@@ -3,11 +3,13 @@
namespace App\Models;
use App\Actions\User\RevokeUserTeamTokens;
use App\Contracts\ThrottledNotification;
use App\Events\ServerReachabilityChanged;
use App\Notifications\Channels\SendsDiscord;
use App\Notifications\Channels\SendsEmail;
use App\Notifications\Channels\SendsPushover;
use App\Notifications\Channels\SendsSlack;
use App\Notifications\Server\Unreachable;
use App\Traits\Auditable;
use App\Traits\HasNotificationSettings;
use App\Traits\HasSafeStringAttribute;
@@ -43,7 +45,10 @@ use OpenApi\Attributes as OA;
class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, SendsSlack
{
use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute, Notifiable;
use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute;
use Notifiable {
notify as sendNotification;
}
protected $fillable = [
'name',
@@ -273,11 +278,23 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen
]);
ServerReachabilityChanged::dispatch($server);
$server->unreachable_count = 3;
$server->unreachable_notification_sent = true;
$server->save();
NotificationThrottle::record($server, Unreachable::class);
}
}
/**
* Send a notification, unless it is throttled and was already sent within its interval.
*/
public function notify($instance): void
{
if ($instance instanceof ThrottledNotification && ! NotificationThrottle::claimFor($instance)) {
return;
}
$this->sendNotification($instance);
}
public function environment_variables()
{
return $this->hasMany(SharedEnvironmentVariable::class)->where('type', 'team');
@@ -2,16 +2,21 @@
namespace App\Notifications\Container;
use App\Contracts\ThrottledNotification;
use App\Models\Server;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class ContainerRestarted extends CustomEmailNotification
class ContainerRestarted extends CustomEmailNotification implements ThrottledNotification
{
public function __construct(public string $name, public Server $server, public ?string $url = null)
/**
* When $restartedResource is set, a restart loop of that resource sends at most one notification per hour.
*/
public function __construct(public string $name, public Server $server, public ?string $url = null, public ?Model $restartedResource = null)
{
$this->onQueue('high');
}
@@ -21,6 +26,16 @@ class ContainerRestarted extends CustomEmailNotification
return $notifiable->getEnabledChannels('status_change');
}
public function throttleSubject(): ?Model
{
return $this->restartedResource;
}
public function throttleIntervalMinutes(): int
{
return 60;
}
public function toMail(): MailMessage
{
$mail = new MailMessage;
+13 -1
View File
@@ -2,14 +2,16 @@
namespace App\Notifications\Server;
use App\Contracts\ThrottledNotification;
use App\Models\Server;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
class HighDiskUsage extends CustomEmailNotification
class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification
{
public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold)
{
@@ -21,6 +23,16 @@ class HighDiskUsage extends CustomEmailNotification
return $notifiable->getEnabledChannels('server_disk_usage');
}
public function throttleSubject(): ?Model
{
return $this->server;
}
public function throttleIntervalMinutes(): int
{
return max(1, (int) data_get($this->server, 'settings.server_disk_usage_notification_interval_hours', 24)) * 60;
}
public function toMail(): MailMessage
{
$mail = new MailMessage;
@@ -0,0 +1,22 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->integer('server_disk_usage_notification_interval_hours')->default(24);
});
}
public function down(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('server_disk_usage_notification_interval_hours');
});
}
};
@@ -0,0 +1,98 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
private const SERVER = 'App\Models\Server';
private const BACKUP = 'App\Models\ScheduledDatabaseBackup';
private const UNREACHABLE = 'App\Notifications\Server\Unreachable';
private const BACKUP_MISSING = 'App\Notifications\Database\BackupMissing';
/**
* Moves the per-resource "notification sent" markers into one table.
*/
public function up(): void
{
Schema::create('notification_throttles', function (Blueprint $table) {
$table->id();
$table->morphs('notifiable');
$table->string('notification');
$table->timestampTz('sent_at');
$table->unique(['notifiable_type', 'notifiable_id', 'notification']);
});
DB::table('servers')
->where('unreachable_notification_sent', true)
->orderBy('id')
->select(['id', 'updated_at'])
->chunk(500, function ($servers) {
DB::table('notification_throttles')->insertOrIgnore($servers->map(fn ($server) => [
'notifiable_type' => self::SERVER,
'notifiable_id' => $server->id,
'notification' => self::UNREACHABLE,
'sent_at' => $server->updated_at ?? now(),
])->all());
});
DB::table('scheduled_database_backups')
->whereNotNull('missing_backup_notification_sent_at')
->orderBy('id')
->select(['id', 'missing_backup_notification_sent_at'])
->chunk(500, function ($backups) {
DB::table('notification_throttles')->insertOrIgnore($backups->map(fn ($backup) => [
'notifiable_type' => self::BACKUP,
'notifiable_id' => $backup->id,
'notification' => self::BACKUP_MISSING,
'sent_at' => $backup->missing_backup_notification_sent_at,
])->all());
});
Schema::table('servers', function (Blueprint $table) {
$table->dropColumn('unreachable_notification_sent');
});
Schema::table('scheduled_database_backups', function (Blueprint $table) {
$table->dropColumn('missing_backup_notification_sent_at');
});
}
public function down(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->boolean('unreachable_notification_sent')->default(false);
});
Schema::table('scheduled_database_backups', function (Blueprint $table) {
$table->timestamp('missing_backup_notification_sent_at')->nullable();
});
DB::table('notification_throttles')
->where('notifiable_type', self::SERVER)
->where('notification', self::UNREACHABLE)
->orderBy('id')
->chunk(500, function ($throttles) {
DB::table('servers')
->whereIn('id', $throttles->pluck('notifiable_id'))
->update(['unreachable_notification_sent' => true]);
});
DB::table('notification_throttles')
->where('notifiable_type', self::BACKUP)
->where('notification', self::BACKUP_MISSING)
->orderBy('id')
->chunk(500, function ($throttles) {
foreach ($throttles as $throttle) {
DB::table('scheduled_database_backups')
->where('id', $throttle->notifiable_id)
->update(['missing_backup_notification_sent_at' => $throttle->sent_at]);
}
});
Schema::dropIfExists('notification_throttles');
}
};
+11 -7
View File
@@ -3531,7 +3531,7 @@
"Applications"
],
"summary": "Get application logs.",
"description": "Get application logs by UUID.",
"description": "Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-application-logs-by-uuid",
"parameters": [
{
@@ -3616,7 +3616,7 @@
"Applications"
],
"summary": "Get preview application logs.",
"description": "Get runtime container logs for a preview deployment by application UUID and pull request ID.",
"description": "Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-preview-application-logs-by-pull-request-id",
"parameters": [
{
@@ -8293,7 +8293,7 @@
"Databases"
],
"summary": "Get database logs.",
"description": "Get database logs by UUID.",
"description": "Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-database-logs-by-uuid",
"parameters": [
{
@@ -16766,6 +16766,10 @@
"type": "string",
"description": "Cron expression for disk usage check frequency."
},
"server_disk_usage_notification_interval_hours": {
"type": "integer",
"description": "Minimum hours between high disk usage notifications (1-720). Default: 24."
},
"connection_timeout": {
"type": "integer",
"description": "SSH connection timeout in seconds (1-300). Default: 10."
@@ -17265,7 +17269,7 @@
"Service applications"
],
"summary": "Get service application logs",
"description": "Get Docker logs for a single compose service container.",
"description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-service-application-logs-by-service-and-app-uuid",
"parameters": [
{
@@ -17350,7 +17354,7 @@
"Service applications"
],
"summary": "Get service application logs",
"description": "Get Docker logs for a single compose service container.",
"description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.",
"operationId": "post-service-application-logs-by-service-and-app-uuid",
"parameters": [
{
@@ -18002,7 +18006,7 @@
"Service databases"
],
"summary": "Get service database logs",
"description": "Get Docker logs for a compose database container.",
"description": "Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-service-database-logs-by-service-and-database-uuid",
"parameters": [
{
@@ -18847,7 +18851,7 @@
"Services"
],
"summary": "Get service logs.",
"description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`.",
"description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-service-logs-by-uuid",
"parameters": [
{
+10 -7
View File
@@ -2325,7 +2325,7 @@ paths:
tags:
- Applications
summary: 'Get application logs.'
description: 'Get application logs by UUID.'
description: 'Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.'
operationId: get-application-logs-by-uuid
parameters:
-
@@ -2383,7 +2383,7 @@ paths:
tags:
- Applications
summary: 'Get preview application logs.'
description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID.'
description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.'
operationId: get-preview-application-logs-by-pull-request-id
parameters:
-
@@ -5469,7 +5469,7 @@ paths:
tags:
- Databases
summary: 'Get database logs.'
description: 'Get database logs by UUID.'
description: 'Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.'
operationId: get-database-logs-by-uuid
parameters:
-
@@ -10647,6 +10647,9 @@ paths:
server_disk_usage_check_frequency:
type: string
description: 'Cron expression for disk usage check frequency.'
server_disk_usage_notification_interval_hours:
type: integer
description: 'Minimum hours between high disk usage notifications (1-720). Default: 24.'
connection_timeout:
type: integer
description: 'SSH connection timeout in seconds (1-300). Default: 10.'
@@ -10931,7 +10934,7 @@ paths:
tags:
- 'Service applications'
summary: 'Get service application logs'
description: 'Get Docker logs for a single compose service container.'
description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'
operationId: get-service-application-logs-by-service-and-app-uuid
parameters:
-
@@ -10989,7 +10992,7 @@ paths:
tags:
- 'Service applications'
summary: 'Get service application logs'
description: 'Get Docker logs for a single compose service container.'
description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'
operationId: post-service-application-logs-by-service-and-app-uuid
parameters:
-
@@ -11418,7 +11421,7 @@ paths:
tags:
- 'Service databases'
summary: 'Get service database logs'
description: 'Get Docker logs for a compose database container.'
description: 'Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.'
operationId: get-service-database-logs-by-service-and-database-uuid
parameters:
-
@@ -11912,7 +11915,7 @@ paths:
tags:
- Services
summary: 'Get service logs.'
description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`.'
description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`. Requires the `read:sensitive` or `root` token ability.'
operationId: get-service-logs-by-uuid
parameters:
-
@@ -14,7 +14,7 @@
<x-application.settings-section id="server-disk-usage-section" title="Disk usage"
helper="Control when Coolify checks this server and when your team is notified.">
<div class="grid gap-4 lg:grid-cols-2">
<div class="grid gap-4 lg:grid-cols-3">
<x-forms.input canGate="update" :canResource="$server" placeholder="0 23 * * *"
id="serverDiskUsageCheckFrequency" label="Check frequency" required
helper="Cron expression or preset such as hourly, daily, weekly, monthly, or yearly." />
@@ -22,6 +22,10 @@
id="serverDiskUsageNotificationThreshold" type="number" min="1" max="99"
label="Notification threshold" required
helper="Notify the team when root filesystem usage exceeds this percentage." />
<x-forms.input canGate="update" :canResource="$server"
id="serverDiskUsageNotificationIntervalHours" type="number" min="1" max="720"
label="Notification interval (hours)" required
helper="Minimum time between high disk usage notifications for this server." />
</div>
</x-application.settings-section>
+5 -5
View File
@@ -7,6 +7,7 @@ use App\Models\CloudInitScript;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\LocalPersistentVolume;
use App\Models\NotificationThrottle;
use App\Models\Project;
use App\Models\ScheduledTask;
use App\Models\Server;
@@ -15,6 +16,7 @@ use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Models\User;
use App\Notifications\Database\BackupMissing;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Queue;
@@ -144,10 +146,8 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {
'frequency' => '0 0 * * *',
'save_s3' => false,
]);
$backup->forceFill([
'last_execution_at' => now()->subDay(),
'missing_backup_notification_sent_at' => now(),
])->save();
$backup->forceFill(['last_execution_at' => now()->subDay()])->save();
NotificationThrottle::record($backup, BackupMissing::class);
$response = $this->withHeaders($this->headers)
->postJson("/api/v1/databases/{$database->uuid}/clone", [
@@ -166,7 +166,7 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {
->and($cloned->destination_id)->toBe($this->destination->id)
->and(str($cloned->status)->startsWith('exited'))->toBeTrue()
->and($clonedBackup->last_execution_at)->toBeNull()
->and($clonedBackup->missing_backup_notification_sent_at)->toBeNull();
->and(NotificationThrottle::wasSent($clonedBackup, BackupMissing::class))->toBeFalse();
});
test('creates renamed volumes when cloning a database with clone_volumes', function () {
@@ -1,7 +1,9 @@
<?php
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Server\Unreachable;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
@@ -13,9 +15,9 @@ it('disables (non-destructively) self-hosted servers with unreachable_count >= 3
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 50,
'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -34,9 +36,9 @@ it('overwrites the IP with 1.2.3.4 on cloud for servers with unreachable_count >
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 50,
'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
NotificationThrottle::record($server, Unreachable::class);
$this->artisan('cleanup:unreachable-servers')->assertSuccessful();
@@ -49,9 +51,9 @@ it('does not clean up servers with unreachable_count less than 3', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 2,
'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -67,9 +69,9 @@ it('does not clean up servers updated within 7 days', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 10,
'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(3),
]);
NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -85,7 +87,6 @@ it('does not clean up servers without notification sent', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 10,
'unreachable_notification_sent' => false,
'updated_at' => now()->subDays(8),
]);
@@ -1,8 +1,10 @@
<?php
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Subscription;
use App\Models\Team;
use App\Notifications\Server\Unreachable;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
@@ -16,7 +18,6 @@ it('sets unreachable fields on servers when subscription ends', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 0,
'unreachable_notification_sent' => false,
]);
$team->subscriptionEnded();
@@ -34,9 +35,9 @@ it('cleans up unsubscribed server IP after 7 days via cleanup command', function
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 3,
'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
NotificationThrottle::record($server, Unreachable::class);
$this->artisan('cleanup:unreachable-servers')->assertSuccessful();
@@ -49,9 +50,9 @@ it('does not clean up unsubscribed server IP within 7 day grace period', functio
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 3,
'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(3),
]);
NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -70,7 +71,6 @@ it('does not affect servers with active subscriptions', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 0,
'unreachable_notification_sent' => false,
]);
$originalCount = $server->unreachable_count;
@@ -1,6 +1,7 @@
<?php
use App\Jobs\CheckMissingDatabaseBackupsJob;
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Team;
@@ -49,7 +50,7 @@ it('notifies the team when an enabled backup has no executions for the configure
(new CheckMissingDatabaseBackupsJob)->handle();
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup));
expect($backup->fresh()->missing_backup_notification_sent_at)->not->toBeNull();
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue();
});
it('does not notify for recent disabled or unconfigured backup schedules', function () {
@@ -114,7 +115,7 @@ it('waits to mark an incident sent until a notification channel is enabled', fun
$backup = missingBackupSchedule($team);
(new CheckMissingDatabaseBackupsJob)->handle();
expect($backup->fresh()->missing_backup_notification_sent_at)->toBeNull();
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
$team->emailNotificationSettings->update([
'smtp_enabled' => true,
@@ -0,0 +1,82 @@
<?php
use App\Models\InstanceSettings;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use App\Notifications\Server\HighDiskUsage;
use App\Notifications\Server\Unreachable;
use Carbon\Carbon;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
$this->team = Team::factory()->create();
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
});
afterEach(function () {
Carbon::setTestNow();
});
it('claims a notification only once until it is released', function () {
expect(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue()
->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse()
->and($this->server->unreachable_notification_sent)->toBeTrue()
->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeTrue()
->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeFalse()
->and($this->server->unreachable_notification_sent)->toBeFalse()
->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue();
});
it('claims a notification again only when it was sent before the given time', function () {
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue();
Carbon::setTestNow(now()->addHours(5));
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeFalse();
Carbon::setTestNow(now()->addHours(2));
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue()
->and(NotificationThrottle::query()->count())->toBe(1);
});
it('keeps throttles separate for each notification and resource', function () {
$otherServer = Server::factory()->create(['team_id' => $this->team->id]);
NotificationThrottle::record($this->server, Unreachable::class);
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class))->toBeTrue()
->and(NotificationThrottle::claim($otherServer, Unreachable::class))->toBeTrue()
->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse();
});
it('removes the throttles of a server when the server is force deleted', function () {
NotificationThrottle::record($this->server, Unreachable::class);
$this->server->forceDelete();
expect(NotificationThrottle::query()->count())->toBe(0);
});
it('keeps unreachable_notification_sent in the server API response', function () {
config()->set('app.maintenance.driver', 'file');
config()->set('cache.default', 'array');
InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);
$user = User::factory()->create();
$this->team->members()->attach($user->id, ['role' => 'owner']);
session(['currentTeam' => $this->team]);
$token = $user->createToken('read-token', ['read']);
$token->accessToken->forceFill(['team_id' => $this->team->id])->save();
NotificationThrottle::record($this->server, Unreachable::class);
$this->withHeaders(['Authorization' => 'Bearer '.$token->plainTextToken])
->getJson('/api/v1/servers/'.$this->server->uuid)
->assertOk()
->assertJsonPath('unreachable_notification_sent', true);
});
@@ -0,0 +1,68 @@
<?php
use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Database\BackupMissing;
use App\Notifications\Server\Unreachable;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
});
it('moves the old notification markers into notification_throttles and back', function () {
$team = Team::factory()->create();
$localhost = Server::factory()->create(['id' => 0, 'team_id' => $team->id]);
$notifiedServer = Server::factory()->create(['team_id' => $team->id]);
$quietServer = Server::factory()->create(['team_id' => $team->id]);
$notifiedBackup = ScheduledDatabaseBackup::create([
'enabled' => true,
'frequency' => '0 0 * * *',
'save_s3' => false,
'database_type' => 'App\\Models\\StandalonePostgresql',
'database_id' => 999,
'team_id' => $team->id,
]);
$quietBackup = $notifiedBackup->replicate(['uuid']);
$quietBackup->save();
$migration = require database_path('migrations/2026_09_29_083244_create_notification_throttles_table.php');
$migration->down();
DB::table('servers')->whereIn('id', [$localhost->id, $notifiedServer->id])->update(['unreachable_notification_sent' => true]);
DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->update(['missing_backup_notification_sent_at' => '2026-09-01 10:00:00']);
$migration->up();
expect(Schema::hasColumn('servers', 'unreachable_notification_sent'))->toBeFalse()
->and(Schema::hasColumn('scheduled_database_backups', 'missing_backup_notification_sent_at'))->toBeFalse()
->and(NotificationThrottle::query()->count())->toBe(3)
->and(NotificationThrottle::wasSent($localhost, Unreachable::class))->toBeTrue()
->and(NotificationThrottle::wasSent($notifiedServer, Unreachable::class))->toBeTrue()
->and(NotificationThrottle::wasSent($quietServer, Unreachable::class))->toBeFalse()
->and(NotificationThrottle::wasSent($quietBackup, BackupMissing::class))->toBeFalse()
->and(NotificationThrottle::query()
->whereMorphedTo('notifiable', $notifiedBackup)
->where('notification', BackupMissing::class)
->sole()
->sent_at
->toDateTimeString())->toBe('2026-09-01 10:00:00');
$migration->down();
expect(Schema::hasTable('notification_throttles'))->toBeFalse()
->and(DB::table('servers')->where('unreachable_notification_sent', true)->pluck('id')->sort()->values()->all())
->toBe([$localhost->id, $notifiedServer->id])
->and(DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->value('missing_backup_notification_sent_at'))
->toStartWith('2026-09-01 10:00:00')
->and(DB::table('scheduled_database_backups')->where('id', $quietBackup->id)->value('missing_backup_notification_sent_at'))
->toBeNull();
$migration->up();
});
@@ -0,0 +1,122 @@
<?php
use App\Jobs\ServerStorageCheckJob;
use App\Livewire\Server\Advanced;
use App\Models\InstanceSettings;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use App\Notifications\Server\HighDiskUsage;
use Carbon\Carbon;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Notification;
use Livewire\Livewire;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
Notification::fake();
InstanceSettings::forceCreate(['id' => 0]);
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
});
afterEach(function () {
Carbon::setTestNow();
});
function diskUsageIntervalServer(int $intervalHours = 24): Server
{
$team = Team::factory()->create();
$team->emailNotificationSettings()->update([
'use_instance_email_settings' => true,
'server_disk_usage_email_notifications' => true,
]);
$server = Server::factory()->create(['team_id' => $team->id, 'ip' => '10.0.0.10']);
$server->settings->update([
'is_reachable' => true,
'is_usable' => true,
'force_disabled' => false,
'server_disk_usage_notification_threshold' => 80,
'server_disk_usage_notification_interval_hours' => $intervalHours,
]);
return $server->fresh();
}
function runDiskUsageCheck(Server $server, int $percentage): void
{
(new ServerStorageCheckJob($server->fresh(), $percentage))->handle();
}
it('sends one high disk usage notification per configured interval', function () {
$server = diskUsageIntervalServer(intervalHours: 6);
runDiskUsageCheck($server, 93);
Carbon::setTestNow(now()->addMinutes(10));
runDiskUsageCheck($server, 93);
Carbon::setTestNow(now()->addHours(5));
runDiskUsageCheck($server, 94);
Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
Carbon::setTestNow(now()->addMinutes(51));
runDiskUsageCheck($server, 94);
Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);
});
it('defaults to one notification per 24 hours', function () {
$team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]);
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
});
it('does not notify below the threshold', function () {
$server = diskUsageIntervalServer();
runDiskUsageCheck($server, 50);
Notification::assertNothingSent();
expect(NotificationThrottle::wasSent($server, HighDiskUsage::class))->toBeFalse();
});
it('saves the notification interval from the server advanced settings', function () {
$team = Team::factory()->create();
$user = User::factory()->create();
$team->members()->attach($user, ['role' => 'owner']);
$server = Server::factory()->create(['team_id' => $team->id]);
$this->actingAs($user);
session(['currentTeam' => $team]);
Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])
->assertSet('serverDiskUsageNotificationIntervalHours', 24)
->set('serverDiskUsageNotificationIntervalHours', 0)
->call('submit')
->assertHasErrors(['serverDiskUsageNotificationIntervalHours'])
->set('serverDiskUsageNotificationIntervalHours', 12)
->call('submit')
->assertHasNoErrors()
->assertDispatched('success');
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(12);
});
it('does not let a member change the notification interval', function () {
$team = Team::factory()->create();
$user = User::factory()->create();
$team->members()->attach($user, ['role' => 'member']);
$server = Server::factory()->create(['team_id' => $team->id]);
$this->actingAs($user);
session(['currentTeam' => $team]);
Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])
->set('serverDiskUsageNotificationIntervalHours', 2)
->call('submit');
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
});
@@ -1,6 +1,7 @@
<?php
use App\Events\ServerReachabilityChanged;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Channels\EmailChannel;
@@ -28,7 +29,6 @@ it('sends Unreachable notification when threshold reached and not yet notified',
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 2,
'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -41,7 +41,6 @@ it('does not send Unreachable on first transient failure (count=1)', function ()
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 1,
'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -53,8 +52,8 @@ it('does not send Unreachable when already notified', function () {
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 5,
'unreachable_notification_sent' => true,
])->save();
NotificationThrottle::record($this->server, Unreachable::class);
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -65,8 +64,8 @@ it('sends Reachable notification on recovery when previously notified', function
$this->server->settings()->update(['is_reachable' => true]);
$this->server->forceFill([
'unreachable_count' => 0,
'unreachable_notification_sent' => true,
])->save();
NotificationThrottle::record($this->server, Unreachable::class);
$fresh = $this->server->fresh();
expect($fresh->unreachable_notification_sent)->toBeTrue();
@@ -82,7 +81,6 @@ it('does not send Reachable when never notified', function () {
$this->server->settings()->update(['is_reachable' => true]);
$this->server->forceFill([
'unreachable_count' => 0,
'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -94,7 +92,6 @@ it('routes Unreachable notification through EmailChannel when email toggle is on
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 2,
'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -227,3 +227,20 @@ it('rejects an invalid disk usage check frequency without partially updating the
->and($server->settings->server_role->value)->toBe('both')
->and($server->proxy->get('type'))->toBe('TRAEFIK');
});
it('updates the disk usage notification interval', function () {
patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [
'server_disk_usage_notification_interval_hours' => 48,
])->assertCreated();
expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(48);
});
it('rejects an out of range disk usage notification interval', function () {
patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [
'server_disk_usage_notification_interval_hours' => 0,
])->assertUnprocessable()
->assertJsonValidationErrors('server_disk_usage_notification_interval_hours');
expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
});
@@ -42,7 +42,6 @@ test('fix mode fully ends a locally active subscription with an invalid Stripe s
$server = Server::factory()->create([
'team_id' => $this->team->id,
'unreachable_count' => 0,
'unreachable_notification_sent' => false,
]);
$stripe = Mockery::mock(StripeClient::class);
+102
View File
@@ -0,0 +1,102 @@
<?php
use App\Jobs\PushServerUpdateJob;
use App\Models\Environment;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Notifications\Container\ContainerRestarted;
use Carbon\Carbon;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
Cache::flush();
Queue::fake();
Notification::fake();
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
$this->team = Team::factory()->create();
$this->team->emailNotificationSettings()->update([
'use_instance_email_settings' => true,
'status_change_email_notifications' => true,
]);
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
});
afterEach(function () {
Carbon::setTestNow();
});
it('sends a throttled notification once per interval for the same resource', function () {
$otherServer = Server::factory()->create(['team_id' => $this->team->id]);
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
$this->team->notify(new ContainerRestarted('coolify-proxy', $otherServer, restartedResource: $otherServer));
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
Carbon::setTestNow(now()->addMinutes(59));
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
Carbon::setTestNow(now()->addMinutes(2));
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 3);
});
it('does not throttle a notification without a throttle subject', function () {
$this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
$this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
});
it('sends one TCP proxy restart notification per hour while the proxy does not stay up', function () {
$project = Project::factory()->create(['team_id' => $this->team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$destination = $this->server->standaloneDockers()->firstOrFail();
$database = StandalonePostgresql::create([
'name' => 'public-db',
'postgres_user' => 'postgres',
'postgres_password' => encrypt('password'),
'postgres_db' => 'app',
'image' => 'postgres:16-alpine',
'is_public' => true,
'public_port' => 54321,
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
]);
// The database runs, but its "-proxy" container is missing on every push.
$push = fn () => (new PushServerUpdateJob($this->server, [
'containers' => [[
'name' => $database->uuid,
'state' => 'running',
'health_status' => 'healthy',
'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid],
]],
'filesystem_usage_root' => ['used_percentage' => 10],
]))->handle();
$push();
Carbon::setTestNow(now()->addMinute());
$push();
Carbon::setTestNow(now()->addMinute());
$push();
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 1);
Carbon::setTestNow(now()->addHour());
$push();
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
});