mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-04 10:42:23 +00:00
fix(notifications): throttle repeated notifications per resource
High disk usage notifications were sent every ~10 minutes on Coolify Cloud because the cache rate limiter did not hold across Sentinel-triggered checks. - Add a notification_throttles table and NotificationThrottle helper that atomically records when a notification was sent for a resource. - Add the ThrottledNotification contract; Team::notify() skips a throttled notification that was already sent within its interval. - High disk usage: configurable per-server interval (server_disk_usage_notification_interval_hours, default 24h) in Server > Advanced and the servers API. - Container restarted (coolify-proxy, database TCP proxy): at most one notification per hour per server/database during a restart loop. - Move the unreachable and missing-backup markers into the new table and migrate existing data (servers.unreachable_notification_sent and scheduled_database_backups.missing_backup_notification_sent_at). - Regenerate OpenAPI files.
This commit is contained in:
1 parent
49086c82a8
commit
99c840922c
33 files changed
+786
-81
No files matched your search
@@ -5,6 +5,8 @@
|
||||
- When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround.
|
||||
- Test old reports against the current branch because later changes can make the report obsolete.
|
||||
- Use the same regression test before and after the production change so the result shows the behavior difference.
|
||||
- Several dev instances can run from one checkout (see `./scripts/dev urls`). After you add a migration, run it on each running instance of that checkout, or those instances fail on the new code.
|
||||
- The checkout is shared with other sessions. When the full suite fails, rerun each failing file alone and compare with a clean `git archive HEAD` copy before you connect a failure to your change.
|
||||
- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.
|
||||
- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Server;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class CleanupUnreachableServers extends Command
|
||||
@@ -14,7 +15,7 @@ class CleanupUnreachableServers extends Command
|
||||
public function handle()
|
||||
{
|
||||
echo "Running unreachable server cleanup...\n";
|
||||
$servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
|
||||
$servers = Server::where('unreachable_count', '>=', 3)->whereHas('notificationThrottles', fn ($query) => $query->where('notification', Unreachable::class))->where('updated_at', '<', now()->subDays(7))->get();
|
||||
if ($servers->count() > 0) {
|
||||
foreach ($servers as $server) {
|
||||
echo "Cleanup unreachable server ($server->id) with name $server->name";
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
<?php
|
||||
|
||||
namespace App\Contracts;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
/**
|
||||
* A notification that is sent at most once per interval for the same resource.
|
||||
* Team::notify() checks the throttle before it sends the notification.
|
||||
*/
|
||||
interface ThrottledNotification
|
||||
{
|
||||
/**
|
||||
* The resource the throttle belongs to. Return null to send without a throttle.
|
||||
*/
|
||||
public function throttleSubject(): ?Model;
|
||||
|
||||
public function throttleIntervalMinutes(): int;
|
||||
}
|
||||
@@ -5119,7 +5119,6 @@ class DatabasesController extends Controller
|
||||
'created_at',
|
||||
'updated_at',
|
||||
'last_execution_at',
|
||||
'missing_backup_notification_sent_at',
|
||||
])->fill([
|
||||
'uuid' => new_public_id(),
|
||||
'database_id' => $newDatabase->id,
|
||||
|
||||
@@ -187,6 +187,7 @@ class ServersController extends Controller
|
||||
$server->load(['settings']);
|
||||
}
|
||||
|
||||
$server->append('unreachable_notification_sent');
|
||||
$settings = $this->removeSensitiveDataFromSettings($server->settings);
|
||||
$server = $this->removeSensitiveData($server);
|
||||
data_set($server, 'settings', $settings);
|
||||
@@ -645,6 +646,7 @@ class ServersController extends Controller
|
||||
'deployment_queue_limit' => ['type' => 'integer', 'description' => 'Maximum number of queued deployments.'],
|
||||
'server_disk_usage_notification_threshold' => ['type' => 'integer', 'description' => 'Server disk usage notification threshold (%).'],
|
||||
'server_disk_usage_check_frequency' => ['type' => 'string', 'description' => 'Cron expression for disk usage check frequency.'],
|
||||
'server_disk_usage_notification_interval_hours' => ['type' => 'integer', 'description' => 'Minimum hours between high disk usage notifications (1-720). Default: 24.'],
|
||||
'connection_timeout' => ['type' => 'integer', 'description' => 'SSH connection timeout in seconds (1-300). Default: 10.'],
|
||||
],
|
||||
),
|
||||
@@ -682,7 +684,7 @@ class ServersController extends Controller
|
||||
)]
|
||||
public function update_server(Request $request)
|
||||
{
|
||||
$allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout', 'is_terminal_enabled'];
|
||||
$allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout', 'is_terminal_enabled'];
|
||||
|
||||
$teamId = getTeamIdFromToken();
|
||||
if (is_null($teamId)) {
|
||||
@@ -708,6 +710,7 @@ class ServersController extends Controller
|
||||
'deployment_queue_limit' => 'integer|min:1',
|
||||
'server_disk_usage_notification_threshold' => 'integer|min:1|max:100',
|
||||
'server_disk_usage_check_frequency' => 'string',
|
||||
'server_disk_usage_notification_interval_hours' => 'integer|min:1|max:720',
|
||||
'connection_timeout' => 'integer|min:1|max:300',
|
||||
'is_terminal_enabled' => 'boolean|nullable',
|
||||
], [
|
||||
@@ -790,7 +793,7 @@ class ServersController extends Controller
|
||||
]);
|
||||
}
|
||||
|
||||
$advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout']);
|
||||
$advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout']);
|
||||
if (! empty($advancedSettings)) {
|
||||
$server->settings()->update(array_filter($advancedSettings, fn ($value) => ! is_null($value)));
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Notifications\Database\BackupMissing;
|
||||
use Illuminate\Bus\Queueable;
|
||||
@@ -39,10 +40,6 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu
|
||||
return;
|
||||
}
|
||||
|
||||
if ($backup->missing_backup_notification_sent_at?->greaterThanOrEqualTo($lastActivityAt)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $backup->team) {
|
||||
Log::warning("Cannot send missing backup notification for backup {$backup->id}: team not found");
|
||||
|
||||
@@ -53,7 +50,11 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu
|
||||
return;
|
||||
}
|
||||
|
||||
// Send once for each period without backup activity.
|
||||
if (! NotificationThrottle::claim($backup, BackupMissing::class, $lastActivityAt)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$backup->team->notify(new BackupMissing($backup, $lastExecutionAt));
|
||||
$backup->forceFill(['missing_backup_notification_sent_at' => now()])->save();
|
||||
}
|
||||
}
|
||||
@@ -799,7 +799,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
try {
|
||||
if (CheckProxy::run($this->server)) {
|
||||
StartProxy::run($this->server, async: false);
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
}
|
||||
@@ -841,7 +841,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
})->first();
|
||||
if (! $tcpProxyContainerFound) {
|
||||
StartDatabaseProxy::dispatch($database);
|
||||
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
|
||||
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database));
|
||||
}
|
||||
} elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
|
||||
// Clean up orphaned proxy containers when is_public=false
|
||||
|
||||
@@ -85,7 +85,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$shouldStart = CheckProxy::run($this->server);
|
||||
if ($shouldStart) {
|
||||
StartProxy::run($this->server, async: false);
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\TimeoutExceededException;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\RateLimiter;
|
||||
use Laravel\Horizon\Contracts\Silenced;
|
||||
|
||||
class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
@@ -31,7 +31,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
|
||||
public function failed(?\Throwable $exception): void
|
||||
{
|
||||
if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
|
||||
if ($exception instanceof TimeoutExceededException) {
|
||||
Log::warning('ServerStorageCheckJob timed out', [
|
||||
'server_id' => $this->server->id,
|
||||
'server_name' => $this->server->name,
|
||||
@@ -58,20 +58,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||
return 'No percentage could be retrieved.';
|
||||
}
|
||||
if ($this->percentage > $serverDiskUsageNotificationThreshold) {
|
||||
$executed = RateLimiter::attempt(
|
||||
'high-disk-usage:'.$this->server->id,
|
||||
$maxAttempts = 0,
|
||||
function () use ($team, $serverDiskUsageNotificationThreshold) {
|
||||
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
|
||||
},
|
||||
$decaySeconds = 3600,
|
||||
);
|
||||
|
||||
if (! $executed) {
|
||||
return 'Too many messages sent!';
|
||||
}
|
||||
} else {
|
||||
RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);
|
||||
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
return handleError($e);
|
||||
|
||||
@@ -18,6 +18,9 @@ class Advanced extends Component
|
||||
#[Validate(['required', 'integer', 'min:1', 'max:99'])]
|
||||
public int|string $serverDiskUsageNotificationThreshold = 50;
|
||||
|
||||
#[Validate(['required', 'integer', 'min:1', 'max:720'])]
|
||||
public int|string $serverDiskUsageNotificationIntervalHours = 24;
|
||||
|
||||
#[Validate(['required', 'integer', 'min:1'])]
|
||||
public int|string $concurrentBuilds = 1;
|
||||
|
||||
@@ -52,6 +55,7 @@ class Advanced extends Component
|
||||
$this->server->settings->backup_compression_cpu_percentage = $this->backupCompressionCpuPercentage;
|
||||
$this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold;
|
||||
$this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency;
|
||||
$this->server->settings->server_disk_usage_notification_interval_hours = $this->serverDiskUsageNotificationIntervalHours;
|
||||
$this->server->settings->save();
|
||||
} else {
|
||||
$this->concurrentBuilds = $this->server->settings->concurrent_builds;
|
||||
@@ -60,6 +64,7 @@ class Advanced extends Component
|
||||
$this->backupCompressionCpuPercentage = $this->server->settings->backup_compression_cpu_percentage;
|
||||
$this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold;
|
||||
$this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency;
|
||||
$this->serverDiskUsageNotificationIntervalHours = $this->server->settings->server_disk_usage_notification_interval_hours;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use App\Contracts\ThrottledNotification;
|
||||
use Carbon\CarbonInterface;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\MorphTo;
|
||||
|
||||
/**
|
||||
* Records when a notification was last sent for a resource, so repeated checks
|
||||
* (across queue workers) do not send the same notification again.
|
||||
*/
|
||||
class NotificationThrottle extends Model
|
||||
{
|
||||
public $timestamps = false;
|
||||
|
||||
protected $fillable = [
|
||||
'notifiable_type',
|
||||
'notifiable_id',
|
||||
'notification',
|
||||
'sent_at',
|
||||
];
|
||||
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'sent_at' => 'datetime',
|
||||
];
|
||||
}
|
||||
|
||||
public function notifiable(): MorphTo
|
||||
{
|
||||
return $this->morphTo();
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically reserve the right to send a notification.
|
||||
*
|
||||
* Returns true when the notification was never sent, or when it was last sent
|
||||
* before $sentBefore. Without $sentBefore, it is sent only once until released.
|
||||
*/
|
||||
public static function claim(Model $notifiable, string $notification, ?CarbonInterface $sentBefore = null): bool
|
||||
{
|
||||
if ($sentBefore !== null) {
|
||||
$renewed = static::query()
|
||||
->whereMorphedTo('notifiable', $notifiable)
|
||||
->where('notification', $notification)
|
||||
->where('sent_at', '<', $sentBefore)
|
||||
->update(['sent_at' => now()]);
|
||||
|
||||
if ($renewed === 1) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return static::query()->insertOrIgnore([
|
||||
'notifiable_type' => $notifiable->getMorphClass(),
|
||||
'notifiable_id' => $notifiable->getKey(),
|
||||
'notification' => $notification,
|
||||
'sent_at' => now(),
|
||||
]) === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Claim the throttle a notification declares. Notifications without a throttle subject are always allowed.
|
||||
*/
|
||||
public static function claimFor(ThrottledNotification $notification): bool
|
||||
{
|
||||
$subject = $notification->throttleSubject();
|
||||
if ($subject === null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return static::claim($subject, $notification::class, now()->subMinutes($notification->throttleIntervalMinutes()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a notification as sent without sending it.
|
||||
*/
|
||||
public static function record(Model $notifiable, string $notification): void
|
||||
{
|
||||
static::query()->updateOrInsert(
|
||||
[
|
||||
'notifiable_type' => $notifiable->getMorphClass(),
|
||||
'notifiable_id' => $notifiable->getKey(),
|
||||
'notification' => $notification,
|
||||
],
|
||||
['sent_at' => now()],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget a sent notification. Returns true when a record was removed.
|
||||
*/
|
||||
public static function release(Model $notifiable, string $notification): bool
|
||||
{
|
||||
return static::query()
|
||||
->whereMorphedTo('notifiable', $notifiable)
|
||||
->where('notification', $notification)
|
||||
->delete() > 0;
|
||||
}
|
||||
|
||||
public static function wasSent(Model $notifiable, string $notification): bool
|
||||
{
|
||||
return static::query()
|
||||
->whereMorphedTo('notifiable', $notifiable)
|
||||
->where('notification', $notification)
|
||||
->exists();
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ class ScheduledDatabaseBackup extends BaseModel
|
||||
'database_backup_retention_max_storage_locally' => 'float',
|
||||
'database_backup_retention_max_storage_s3' => 'float',
|
||||
'missing_backup_notification_days' => 'integer',
|
||||
'missing_backup_notification_sent_at' => 'datetime',
|
||||
'last_execution_at' => 'datetime',
|
||||
'next_run_at' => 'datetime',
|
||||
];
|
||||
|
||||
+21
-9
@@ -29,6 +29,7 @@ use App\Traits\HasSafeStringAttribute;
|
||||
use Illuminate\Database\Eloquent\Builder;
|
||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Relations\MorphMany;
|
||||
use Illuminate\Database\Eloquent\SoftDeletes;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Collection;
|
||||
@@ -235,6 +236,7 @@ class Server extends BaseModel
|
||||
});
|
||||
$server->settings()->delete();
|
||||
$server->sslCertificates()->delete();
|
||||
$server->notificationThrottles()->delete();
|
||||
});
|
||||
|
||||
static::updated(function () {
|
||||
@@ -280,7 +282,6 @@ class Server extends BaseModel
|
||||
'logdrain_newrelic_license_key' => 'encrypted',
|
||||
'delete_unused_volumes' => 'boolean',
|
||||
'delete_unused_networks' => 'boolean',
|
||||
'unreachable_notification_sent' => 'boolean',
|
||||
'force_disabled' => 'boolean',
|
||||
'sentinel_waiting_since' => 'datetime',
|
||||
];
|
||||
@@ -1526,6 +1527,19 @@ $siteAddress {
|
||||
return $this->belongsTo(CloudProviderToken::class);
|
||||
}
|
||||
|
||||
public function notificationThrottles(): MorphMany
|
||||
{
|
||||
return $this->morphMany(NotificationThrottle::class, 'notifiable');
|
||||
}
|
||||
|
||||
/**
|
||||
* True while an Unreachable notification was sent and no Reachable notification followed.
|
||||
*/
|
||||
protected function unreachableNotificationSent(): Attribute
|
||||
{
|
||||
return Attribute::get(fn (): bool => NotificationThrottle::wasSent($this, Unreachable::class));
|
||||
}
|
||||
|
||||
public function sslCertificates()
|
||||
{
|
||||
return $this->hasMany(SslCertificate::class);
|
||||
@@ -1777,18 +1791,16 @@ $siteAddress {
|
||||
|
||||
public function sendReachableNotification()
|
||||
{
|
||||
$this->unreachable_notification_sent = false;
|
||||
$this->save();
|
||||
$this->refresh();
|
||||
$this->team->notify(new Reachable($this));
|
||||
if (NotificationThrottle::release($this, Unreachable::class)) {
|
||||
$this->team->notify(new Reachable($this));
|
||||
}
|
||||
}
|
||||
|
||||
public function sendUnreachableNotification()
|
||||
{
|
||||
$this->unreachable_notification_sent = true;
|
||||
$this->save();
|
||||
$this->refresh();
|
||||
$this->team->notify(new Unreachable($this));
|
||||
if (NotificationThrottle::claim($this, Unreachable::class)) {
|
||||
$this->team->notify(new Unreachable($this));
|
||||
}
|
||||
}
|
||||
|
||||
public function validateConnection(bool $justCheckingNewKey = false)
|
||||
|
||||
@@ -117,6 +117,7 @@ class ServerSetting extends Model
|
||||
'server_disk_usage_notification_threshold',
|
||||
'is_sentinel_debug_enabled',
|
||||
'server_disk_usage_check_frequency',
|
||||
'server_disk_usage_notification_interval_hours',
|
||||
'is_terminal_enabled',
|
||||
'deployment_queue_limit',
|
||||
'backup_compression_cpu_percentage',
|
||||
@@ -170,6 +171,7 @@ class ServerSetting extends Model
|
||||
'compose_version_checked_at' => 'datetime',
|
||||
'backup_compression_cpu_percentage' => 'integer',
|
||||
'docker_cleanup_next_run_at' => 'datetime',
|
||||
'server_disk_usage_notification_interval_hours' => 'integer',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
+19
-2
@@ -3,11 +3,13 @@
|
||||
namespace App\Models;
|
||||
|
||||
use App\Actions\User\RevokeUserTeamTokens;
|
||||
use App\Contracts\ThrottledNotification;
|
||||
use App\Events\ServerReachabilityChanged;
|
||||
use App\Notifications\Channels\SendsDiscord;
|
||||
use App\Notifications\Channels\SendsEmail;
|
||||
use App\Notifications\Channels\SendsPushover;
|
||||
use App\Notifications\Channels\SendsSlack;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use App\Traits\Auditable;
|
||||
use App\Traits\HasNotificationSettings;
|
||||
use App\Traits\HasSafeStringAttribute;
|
||||
@@ -43,7 +45,10 @@ use OpenApi\Attributes as OA;
|
||||
|
||||
class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, SendsSlack
|
||||
{
|
||||
use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute, Notifiable;
|
||||
use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute;
|
||||
use Notifiable {
|
||||
notify as sendNotification;
|
||||
}
|
||||
|
||||
protected $fillable = [
|
||||
'name',
|
||||
@@ -273,11 +278,23 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen
|
||||
]);
|
||||
ServerReachabilityChanged::dispatch($server);
|
||||
$server->unreachable_count = 3;
|
||||
$server->unreachable_notification_sent = true;
|
||||
$server->save();
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a notification, unless it is throttled and was already sent within its interval.
|
||||
*/
|
||||
public function notify($instance): void
|
||||
{
|
||||
if ($instance instanceof ThrottledNotification && ! NotificationThrottle::claimFor($instance)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->sendNotification($instance);
|
||||
}
|
||||
|
||||
public function environment_variables()
|
||||
{
|
||||
return $this->hasMany(SharedEnvironmentVariable::class)->where('type', 'team');
|
||||
|
||||
@@ -2,16 +2,21 @@
|
||||
|
||||
namespace App\Notifications\Container;
|
||||
|
||||
use App\Contracts\ThrottledNotification;
|
||||
use App\Models\Server;
|
||||
use App\Notifications\CustomEmailNotification;
|
||||
use App\Notifications\Dto\DiscordMessage;
|
||||
use App\Notifications\Dto\PushoverMessage;
|
||||
use App\Notifications\Dto\SlackMessage;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Notifications\Messages\MailMessage;
|
||||
|
||||
class ContainerRestarted extends CustomEmailNotification
|
||||
class ContainerRestarted extends CustomEmailNotification implements ThrottledNotification
|
||||
{
|
||||
public function __construct(public string $name, public Server $server, public ?string $url = null)
|
||||
/**
|
||||
* When $restartedResource is set, a restart loop of that resource sends at most one notification per hour.
|
||||
*/
|
||||
public function __construct(public string $name, public Server $server, public ?string $url = null, public ?Model $restartedResource = null)
|
||||
{
|
||||
$this->onQueue('high');
|
||||
}
|
||||
@@ -21,6 +26,16 @@ class ContainerRestarted extends CustomEmailNotification
|
||||
return $notifiable->getEnabledChannels('status_change');
|
||||
}
|
||||
|
||||
public function throttleSubject(): ?Model
|
||||
{
|
||||
return $this->restartedResource;
|
||||
}
|
||||
|
||||
public function throttleIntervalMinutes(): int
|
||||
{
|
||||
return 60;
|
||||
}
|
||||
|
||||
public function toMail(): MailMessage
|
||||
{
|
||||
$mail = new MailMessage;
|
||||
|
||||
@@ -2,14 +2,16 @@
|
||||
|
||||
namespace App\Notifications\Server;
|
||||
|
||||
use App\Contracts\ThrottledNotification;
|
||||
use App\Models\Server;
|
||||
use App\Notifications\CustomEmailNotification;
|
||||
use App\Notifications\Dto\DiscordMessage;
|
||||
use App\Notifications\Dto\PushoverMessage;
|
||||
use App\Notifications\Dto\SlackMessage;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Notifications\Messages\MailMessage;
|
||||
|
||||
class HighDiskUsage extends CustomEmailNotification
|
||||
class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification
|
||||
{
|
||||
public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold)
|
||||
{
|
||||
@@ -21,6 +23,16 @@ class HighDiskUsage extends CustomEmailNotification
|
||||
return $notifiable->getEnabledChannels('server_disk_usage');
|
||||
}
|
||||
|
||||
public function throttleSubject(): ?Model
|
||||
{
|
||||
return $this->server;
|
||||
}
|
||||
|
||||
public function throttleIntervalMinutes(): int
|
||||
{
|
||||
return max(1, (int) data_get($this->server, 'settings.server_disk_usage_notification_interval_hours', 24)) * 60;
|
||||
}
|
||||
|
||||
public function toMail(): MailMessage
|
||||
{
|
||||
$mail = new MailMessage;
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('server_settings', function (Blueprint $table) {
|
||||
$table->integer('server_disk_usage_notification_interval_hours')->default(24);
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('server_settings', function (Blueprint $table) {
|
||||
$table->dropColumn('server_disk_usage_notification_interval_hours');
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,98 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
private const SERVER = 'App\Models\Server';
|
||||
|
||||
private const BACKUP = 'App\Models\ScheduledDatabaseBackup';
|
||||
|
||||
private const UNREACHABLE = 'App\Notifications\Server\Unreachable';
|
||||
|
||||
private const BACKUP_MISSING = 'App\Notifications\Database\BackupMissing';
|
||||
|
||||
/**
|
||||
* Moves the per-resource "notification sent" markers into one table.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('notification_throttles', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->morphs('notifiable');
|
||||
$table->string('notification');
|
||||
$table->timestampTz('sent_at');
|
||||
$table->unique(['notifiable_type', 'notifiable_id', 'notification']);
|
||||
});
|
||||
|
||||
DB::table('servers')
|
||||
->where('unreachable_notification_sent', true)
|
||||
->orderBy('id')
|
||||
->select(['id', 'updated_at'])
|
||||
->chunk(500, function ($servers) {
|
||||
DB::table('notification_throttles')->insertOrIgnore($servers->map(fn ($server) => [
|
||||
'notifiable_type' => self::SERVER,
|
||||
'notifiable_id' => $server->id,
|
||||
'notification' => self::UNREACHABLE,
|
||||
'sent_at' => $server->updated_at ?? now(),
|
||||
])->all());
|
||||
});
|
||||
|
||||
DB::table('scheduled_database_backups')
|
||||
->whereNotNull('missing_backup_notification_sent_at')
|
||||
->orderBy('id')
|
||||
->select(['id', 'missing_backup_notification_sent_at'])
|
||||
->chunk(500, function ($backups) {
|
||||
DB::table('notification_throttles')->insertOrIgnore($backups->map(fn ($backup) => [
|
||||
'notifiable_type' => self::BACKUP,
|
||||
'notifiable_id' => $backup->id,
|
||||
'notification' => self::BACKUP_MISSING,
|
||||
'sent_at' => $backup->missing_backup_notification_sent_at,
|
||||
])->all());
|
||||
});
|
||||
|
||||
Schema::table('servers', function (Blueprint $table) {
|
||||
$table->dropColumn('unreachable_notification_sent');
|
||||
});
|
||||
Schema::table('scheduled_database_backups', function (Blueprint $table) {
|
||||
$table->dropColumn('missing_backup_notification_sent_at');
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('servers', function (Blueprint $table) {
|
||||
$table->boolean('unreachable_notification_sent')->default(false);
|
||||
});
|
||||
Schema::table('scheduled_database_backups', function (Blueprint $table) {
|
||||
$table->timestamp('missing_backup_notification_sent_at')->nullable();
|
||||
});
|
||||
|
||||
DB::table('notification_throttles')
|
||||
->where('notifiable_type', self::SERVER)
|
||||
->where('notification', self::UNREACHABLE)
|
||||
->orderBy('id')
|
||||
->chunk(500, function ($throttles) {
|
||||
DB::table('servers')
|
||||
->whereIn('id', $throttles->pluck('notifiable_id'))
|
||||
->update(['unreachable_notification_sent' => true]);
|
||||
});
|
||||
|
||||
DB::table('notification_throttles')
|
||||
->where('notifiable_type', self::BACKUP)
|
||||
->where('notification', self::BACKUP_MISSING)
|
||||
->orderBy('id')
|
||||
->chunk(500, function ($throttles) {
|
||||
foreach ($throttles as $throttle) {
|
||||
DB::table('scheduled_database_backups')
|
||||
->where('id', $throttle->notifiable_id)
|
||||
->update(['missing_backup_notification_sent_at' => $throttle->sent_at]);
|
||||
}
|
||||
});
|
||||
|
||||
Schema::dropIfExists('notification_throttles');
|
||||
}
|
||||
};
|
||||
+11
-7
@@ -3531,7 +3531,7 @@
|
||||
"Applications"
|
||||
],
|
||||
"summary": "Get application logs.",
|
||||
"description": "Get application logs by UUID.",
|
||||
"description": "Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "get-application-logs-by-uuid",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -3616,7 +3616,7 @@
|
||||
"Applications"
|
||||
],
|
||||
"summary": "Get preview application logs.",
|
||||
"description": "Get runtime container logs for a preview deployment by application UUID and pull request ID.",
|
||||
"description": "Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "get-preview-application-logs-by-pull-request-id",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -8293,7 +8293,7 @@
|
||||
"Databases"
|
||||
],
|
||||
"summary": "Get database logs.",
|
||||
"description": "Get database logs by UUID.",
|
||||
"description": "Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "get-database-logs-by-uuid",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -16766,6 +16766,10 @@
|
||||
"type": "string",
|
||||
"description": "Cron expression for disk usage check frequency."
|
||||
},
|
||||
"server_disk_usage_notification_interval_hours": {
|
||||
"type": "integer",
|
||||
"description": "Minimum hours between high disk usage notifications (1-720). Default: 24."
|
||||
},
|
||||
"connection_timeout": {
|
||||
"type": "integer",
|
||||
"description": "SSH connection timeout in seconds (1-300). Default: 10."
|
||||
@@ -17265,7 +17269,7 @@
|
||||
"Service applications"
|
||||
],
|
||||
"summary": "Get service application logs",
|
||||
"description": "Get Docker logs for a single compose service container.",
|
||||
"description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "get-service-application-logs-by-service-and-app-uuid",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -17350,7 +17354,7 @@
|
||||
"Service applications"
|
||||
],
|
||||
"summary": "Get service application logs",
|
||||
"description": "Get Docker logs for a single compose service container.",
|
||||
"description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "post-service-application-logs-by-service-and-app-uuid",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -18002,7 +18006,7 @@
|
||||
"Service databases"
|
||||
],
|
||||
"summary": "Get service database logs",
|
||||
"description": "Get Docker logs for a compose database container.",
|
||||
"description": "Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "get-service-database-logs-by-service-and-database-uuid",
|
||||
"parameters": [
|
||||
{
|
||||
@@ -18847,7 +18851,7 @@
|
||||
"Services"
|
||||
],
|
||||
"summary": "Get service logs.",
|
||||
"description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`.",
|
||||
"description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`. Requires the `read:sensitive` or `root` token ability.",
|
||||
"operationId": "get-service-logs-by-uuid",
|
||||
"parameters": [
|
||||
{
|
||||
|
||||
+10
-7
@@ -2325,7 +2325,7 @@ paths:
|
||||
tags:
|
||||
- Applications
|
||||
summary: 'Get application logs.'
|
||||
description: 'Get application logs by UUID.'
|
||||
description: 'Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: get-application-logs-by-uuid
|
||||
parameters:
|
||||
-
|
||||
@@ -2383,7 +2383,7 @@ paths:
|
||||
tags:
|
||||
- Applications
|
||||
summary: 'Get preview application logs.'
|
||||
description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID.'
|
||||
description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: get-preview-application-logs-by-pull-request-id
|
||||
parameters:
|
||||
-
|
||||
@@ -5469,7 +5469,7 @@ paths:
|
||||
tags:
|
||||
- Databases
|
||||
summary: 'Get database logs.'
|
||||
description: 'Get database logs by UUID.'
|
||||
description: 'Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: get-database-logs-by-uuid
|
||||
parameters:
|
||||
-
|
||||
@@ -10647,6 +10647,9 @@ paths:
|
||||
server_disk_usage_check_frequency:
|
||||
type: string
|
||||
description: 'Cron expression for disk usage check frequency.'
|
||||
server_disk_usage_notification_interval_hours:
|
||||
type: integer
|
||||
description: 'Minimum hours between high disk usage notifications (1-720). Default: 24.'
|
||||
connection_timeout:
|
||||
type: integer
|
||||
description: 'SSH connection timeout in seconds (1-300). Default: 10.'
|
||||
@@ -10931,7 +10934,7 @@ paths:
|
||||
tags:
|
||||
- 'Service applications'
|
||||
summary: 'Get service application logs'
|
||||
description: 'Get Docker logs for a single compose service container.'
|
||||
description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: get-service-application-logs-by-service-and-app-uuid
|
||||
parameters:
|
||||
-
|
||||
@@ -10989,7 +10992,7 @@ paths:
|
||||
tags:
|
||||
- 'Service applications'
|
||||
summary: 'Get service application logs'
|
||||
description: 'Get Docker logs for a single compose service container.'
|
||||
description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: post-service-application-logs-by-service-and-app-uuid
|
||||
parameters:
|
||||
-
|
||||
@@ -11418,7 +11421,7 @@ paths:
|
||||
tags:
|
||||
- 'Service databases'
|
||||
summary: 'Get service database logs'
|
||||
description: 'Get Docker logs for a compose database container.'
|
||||
description: 'Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: get-service-database-logs-by-service-and-database-uuid
|
||||
parameters:
|
||||
-
|
||||
@@ -11912,7 +11915,7 @@ paths:
|
||||
tags:
|
||||
- Services
|
||||
summary: 'Get service logs.'
|
||||
description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`.'
|
||||
description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`. Requires the `read:sensitive` or `root` token ability.'
|
||||
operationId: get-service-logs-by-uuid
|
||||
parameters:
|
||||
-
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
<x-application.settings-section id="server-disk-usage-section" title="Disk usage"
|
||||
helper="Control when Coolify checks this server and when your team is notified.">
|
||||
<div class="grid gap-4 lg:grid-cols-2">
|
||||
<div class="grid gap-4 lg:grid-cols-3">
|
||||
<x-forms.input canGate="update" :canResource="$server" placeholder="0 23 * * *"
|
||||
id="serverDiskUsageCheckFrequency" label="Check frequency" required
|
||||
helper="Cron expression or preset such as hourly, daily, weekly, monthly, or yearly." />
|
||||
@@ -22,6 +22,10 @@
|
||||
id="serverDiskUsageNotificationThreshold" type="number" min="1" max="99"
|
||||
label="Notification threshold" required
|
||||
helper="Notify the team when root filesystem usage exceeds this percentage." />
|
||||
<x-forms.input canGate="update" :canResource="$server"
|
||||
id="serverDiskUsageNotificationIntervalHours" type="number" min="1" max="720"
|
||||
label="Notification interval (hours)" required
|
||||
helper="Minimum time between high disk usage notifications for this server." />
|
||||
</div>
|
||||
</x-application.settings-section>
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ use App\Models\CloudInitScript;
|
||||
use App\Models\Environment;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\LocalPersistentVolume;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\Project;
|
||||
use App\Models\ScheduledTask;
|
||||
use App\Models\Server;
|
||||
@@ -15,6 +16,7 @@ use App\Models\StandaloneDocker;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\Database\BackupMissing;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Bus;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
@@ -144,10 +146,8 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {
|
||||
'frequency' => '0 0 * * *',
|
||||
'save_s3' => false,
|
||||
]);
|
||||
$backup->forceFill([
|
||||
'last_execution_at' => now()->subDay(),
|
||||
'missing_backup_notification_sent_at' => now(),
|
||||
])->save();
|
||||
$backup->forceFill(['last_execution_at' => now()->subDay()])->save();
|
||||
NotificationThrottle::record($backup, BackupMissing::class);
|
||||
|
||||
$response = $this->withHeaders($this->headers)
|
||||
->postJson("/api/v1/databases/{$database->uuid}/clone", [
|
||||
@@ -166,7 +166,7 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {
|
||||
->and($cloned->destination_id)->toBe($this->destination->id)
|
||||
->and(str($cloned->status)->startsWith('exited'))->toBeTrue()
|
||||
->and($clonedBackup->last_execution_at)->toBeNull()
|
||||
->and($clonedBackup->missing_backup_notification_sent_at)->toBeNull();
|
||||
->and(NotificationThrottle::wasSent($clonedBackup, BackupMissing::class))->toBeFalse();
|
||||
});
|
||||
|
||||
test('creates renamed volumes when cloning a database with clone_volumes', function () {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
<?php
|
||||
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
@@ -13,9 +15,9 @@ it('disables (non-destructively) self-hosted servers with unreachable_count >= 3
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 50,
|
||||
'unreachable_notification_sent' => true,
|
||||
'updated_at' => now()->subDays(8),
|
||||
]);
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
|
||||
$originalIp = (string) $server->ip;
|
||||
|
||||
@@ -34,9 +36,9 @@ it('overwrites the IP with 1.2.3.4 on cloud for servers with unreachable_count >
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 50,
|
||||
'unreachable_notification_sent' => true,
|
||||
'updated_at' => now()->subDays(8),
|
||||
]);
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
|
||||
$this->artisan('cleanup:unreachable-servers')->assertSuccessful();
|
||||
|
||||
@@ -49,9 +51,9 @@ it('does not clean up servers with unreachable_count less than 3', function () {
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 2,
|
||||
'unreachable_notification_sent' => true,
|
||||
'updated_at' => now()->subDays(8),
|
||||
]);
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
|
||||
$originalIp = (string) $server->ip;
|
||||
|
||||
@@ -67,9 +69,9 @@ it('does not clean up servers updated within 7 days', function () {
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 10,
|
||||
'unreachable_notification_sent' => true,
|
||||
'updated_at' => now()->subDays(3),
|
||||
]);
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
|
||||
$originalIp = (string) $server->ip;
|
||||
|
||||
@@ -85,7 +87,6 @@ it('does not clean up servers without notification sent', function () {
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 10,
|
||||
'unreachable_notification_sent' => false,
|
||||
'updated_at' => now()->subDays(8),
|
||||
]);
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
<?php
|
||||
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\Server;
|
||||
use App\Models\Subscription;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
@@ -16,7 +18,6 @@ it('sets unreachable fields on servers when subscription ends', function () {
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 0,
|
||||
'unreachable_notification_sent' => false,
|
||||
]);
|
||||
|
||||
$team->subscriptionEnded();
|
||||
@@ -34,9 +35,9 @@ it('cleans up unsubscribed server IP after 7 days via cleanup command', function
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 3,
|
||||
'unreachable_notification_sent' => true,
|
||||
'updated_at' => now()->subDays(8),
|
||||
]);
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
|
||||
$this->artisan('cleanup:unreachable-servers')->assertSuccessful();
|
||||
|
||||
@@ -49,9 +50,9 @@ it('does not clean up unsubscribed server IP within 7 day grace period', functio
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 3,
|
||||
'unreachable_notification_sent' => true,
|
||||
'updated_at' => now()->subDays(3),
|
||||
]);
|
||||
NotificationThrottle::record($server, Unreachable::class);
|
||||
|
||||
$originalIp = (string) $server->ip;
|
||||
|
||||
@@ -70,7 +71,6 @@ it('does not affect servers with active subscriptions', function () {
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'unreachable_count' => 0,
|
||||
'unreachable_notification_sent' => false,
|
||||
]);
|
||||
|
||||
$originalCount = $server->unreachable_count;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CheckMissingDatabaseBackupsJob;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledDatabaseBackupExecution;
|
||||
use App\Models\Team;
|
||||
@@ -49,7 +50,7 @@ it('notifies the team when an enabled backup has no executions for the configure
|
||||
(new CheckMissingDatabaseBackupsJob)->handle();
|
||||
|
||||
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup));
|
||||
expect($backup->fresh()->missing_backup_notification_sent_at)->not->toBeNull();
|
||||
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue();
|
||||
});
|
||||
|
||||
it('does not notify for recent disabled or unconfigured backup schedules', function () {
|
||||
@@ -114,7 +115,7 @@ it('waits to mark an incident sent until a notification channel is enabled', fun
|
||||
$backup = missingBackupSchedule($team);
|
||||
|
||||
(new CheckMissingDatabaseBackupsJob)->handle();
|
||||
expect($backup->fresh()->missing_backup_notification_sent_at)->toBeNull();
|
||||
expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
|
||||
|
||||
$team->emailNotificationSettings->update([
|
||||
'smtp_enabled' => true,
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
<?php
|
||||
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\Server\HighDiskUsage;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
$this->team = Team::factory()->create();
|
||||
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
Carbon::setTestNow();
|
||||
});
|
||||
|
||||
it('claims a notification only once until it is released', function () {
|
||||
expect(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue()
|
||||
->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse()
|
||||
->and($this->server->unreachable_notification_sent)->toBeTrue()
|
||||
->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeTrue()
|
||||
->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeFalse()
|
||||
->and($this->server->unreachable_notification_sent)->toBeFalse()
|
||||
->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue();
|
||||
});
|
||||
|
||||
it('claims a notification again only when it was sent before the given time', function () {
|
||||
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
|
||||
|
||||
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue();
|
||||
|
||||
Carbon::setTestNow(now()->addHours(5));
|
||||
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeFalse();
|
||||
|
||||
Carbon::setTestNow(now()->addHours(2));
|
||||
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue()
|
||||
->and(NotificationThrottle::query()->count())->toBe(1);
|
||||
});
|
||||
|
||||
it('keeps throttles separate for each notification and resource', function () {
|
||||
$otherServer = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
|
||||
NotificationThrottle::record($this->server, Unreachable::class);
|
||||
|
||||
expect(NotificationThrottle::claim($this->server, HighDiskUsage::class))->toBeTrue()
|
||||
->and(NotificationThrottle::claim($otherServer, Unreachable::class))->toBeTrue()
|
||||
->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse();
|
||||
});
|
||||
|
||||
it('removes the throttles of a server when the server is force deleted', function () {
|
||||
NotificationThrottle::record($this->server, Unreachable::class);
|
||||
|
||||
$this->server->forceDelete();
|
||||
|
||||
expect(NotificationThrottle::query()->count())->toBe(0);
|
||||
});
|
||||
|
||||
it('keeps unreachable_notification_sent in the server API response', function () {
|
||||
config()->set('app.maintenance.driver', 'file');
|
||||
config()->set('cache.default', 'array');
|
||||
InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);
|
||||
$user = User::factory()->create();
|
||||
$this->team->members()->attach($user->id, ['role' => 'owner']);
|
||||
session(['currentTeam' => $this->team]);
|
||||
$token = $user->createToken('read-token', ['read']);
|
||||
$token->accessToken->forceFill(['team_id' => $this->team->id])->save();
|
||||
|
||||
NotificationThrottle::record($this->server, Unreachable::class);
|
||||
|
||||
$this->withHeaders(['Authorization' => 'Bearer '.$token->plainTextToken])
|
||||
->getJson('/api/v1/servers/'.$this->server->uuid)
|
||||
->assertOk()
|
||||
->assertJsonPath('unreachable_notification_sent', true);
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Database\BackupMissing;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
});
|
||||
|
||||
it('moves the old notification markers into notification_throttles and back', function () {
|
||||
$team = Team::factory()->create();
|
||||
$localhost = Server::factory()->create(['id' => 0, 'team_id' => $team->id]);
|
||||
$notifiedServer = Server::factory()->create(['team_id' => $team->id]);
|
||||
$quietServer = Server::factory()->create(['team_id' => $team->id]);
|
||||
$notifiedBackup = ScheduledDatabaseBackup::create([
|
||||
'enabled' => true,
|
||||
'frequency' => '0 0 * * *',
|
||||
'save_s3' => false,
|
||||
'database_type' => 'App\\Models\\StandalonePostgresql',
|
||||
'database_id' => 999,
|
||||
'team_id' => $team->id,
|
||||
]);
|
||||
$quietBackup = $notifiedBackup->replicate(['uuid']);
|
||||
$quietBackup->save();
|
||||
|
||||
$migration = require database_path('migrations/2026_09_29_083244_create_notification_throttles_table.php');
|
||||
$migration->down();
|
||||
|
||||
DB::table('servers')->whereIn('id', [$localhost->id, $notifiedServer->id])->update(['unreachable_notification_sent' => true]);
|
||||
DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->update(['missing_backup_notification_sent_at' => '2026-09-01 10:00:00']);
|
||||
|
||||
$migration->up();
|
||||
|
||||
expect(Schema::hasColumn('servers', 'unreachable_notification_sent'))->toBeFalse()
|
||||
->and(Schema::hasColumn('scheduled_database_backups', 'missing_backup_notification_sent_at'))->toBeFalse()
|
||||
->and(NotificationThrottle::query()->count())->toBe(3)
|
||||
->and(NotificationThrottle::wasSent($localhost, Unreachable::class))->toBeTrue()
|
||||
->and(NotificationThrottle::wasSent($notifiedServer, Unreachable::class))->toBeTrue()
|
||||
->and(NotificationThrottle::wasSent($quietServer, Unreachable::class))->toBeFalse()
|
||||
->and(NotificationThrottle::wasSent($quietBackup, BackupMissing::class))->toBeFalse()
|
||||
->and(NotificationThrottle::query()
|
||||
->whereMorphedTo('notifiable', $notifiedBackup)
|
||||
->where('notification', BackupMissing::class)
|
||||
->sole()
|
||||
->sent_at
|
||||
->toDateTimeString())->toBe('2026-09-01 10:00:00');
|
||||
|
||||
$migration->down();
|
||||
|
||||
expect(Schema::hasTable('notification_throttles'))->toBeFalse()
|
||||
->and(DB::table('servers')->where('unreachable_notification_sent', true)->pluck('id')->sort()->values()->all())
|
||||
->toBe([$localhost->id, $notifiedServer->id])
|
||||
->and(DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->value('missing_backup_notification_sent_at'))
|
||||
->toStartWith('2026-09-01 10:00:00')
|
||||
->and(DB::table('scheduled_database_backups')->where('id', $quietBackup->id)->value('missing_backup_notification_sent_at'))
|
||||
->toBeNull();
|
||||
|
||||
$migration->up();
|
||||
});
|
||||
@@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\ServerStorageCheckJob;
|
||||
use App\Livewire\Server\Advanced;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\Server\HighDiskUsage;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
use Livewire\Livewire;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
Notification::fake();
|
||||
InstanceSettings::forceCreate(['id' => 0]);
|
||||
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
Carbon::setTestNow();
|
||||
});
|
||||
|
||||
function diskUsageIntervalServer(int $intervalHours = 24): Server
|
||||
{
|
||||
$team = Team::factory()->create();
|
||||
$team->emailNotificationSettings()->update([
|
||||
'use_instance_email_settings' => true,
|
||||
'server_disk_usage_email_notifications' => true,
|
||||
]);
|
||||
$server = Server::factory()->create(['team_id' => $team->id, 'ip' => '10.0.0.10']);
|
||||
$server->settings->update([
|
||||
'is_reachable' => true,
|
||||
'is_usable' => true,
|
||||
'force_disabled' => false,
|
||||
'server_disk_usage_notification_threshold' => 80,
|
||||
'server_disk_usage_notification_interval_hours' => $intervalHours,
|
||||
]);
|
||||
|
||||
return $server->fresh();
|
||||
}
|
||||
|
||||
function runDiskUsageCheck(Server $server, int $percentage): void
|
||||
{
|
||||
(new ServerStorageCheckJob($server->fresh(), $percentage))->handle();
|
||||
}
|
||||
|
||||
it('sends one high disk usage notification per configured interval', function () {
|
||||
$server = diskUsageIntervalServer(intervalHours: 6);
|
||||
|
||||
runDiskUsageCheck($server, 93);
|
||||
Carbon::setTestNow(now()->addMinutes(10));
|
||||
runDiskUsageCheck($server, 93);
|
||||
Carbon::setTestNow(now()->addHours(5));
|
||||
runDiskUsageCheck($server, 94);
|
||||
|
||||
Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
|
||||
|
||||
Carbon::setTestNow(now()->addMinutes(51));
|
||||
runDiskUsageCheck($server, 94);
|
||||
|
||||
Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);
|
||||
});
|
||||
|
||||
it('defaults to one notification per 24 hours', function () {
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create(['team_id' => $team->id]);
|
||||
|
||||
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
|
||||
});
|
||||
|
||||
it('does not notify below the threshold', function () {
|
||||
$server = diskUsageIntervalServer();
|
||||
|
||||
runDiskUsageCheck($server, 50);
|
||||
|
||||
Notification::assertNothingSent();
|
||||
expect(NotificationThrottle::wasSent($server, HighDiskUsage::class))->toBeFalse();
|
||||
});
|
||||
|
||||
it('saves the notification interval from the server advanced settings', function () {
|
||||
$team = Team::factory()->create();
|
||||
$user = User::factory()->create();
|
||||
$team->members()->attach($user, ['role' => 'owner']);
|
||||
$server = Server::factory()->create(['team_id' => $team->id]);
|
||||
|
||||
$this->actingAs($user);
|
||||
session(['currentTeam' => $team]);
|
||||
|
||||
Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])
|
||||
->assertSet('serverDiskUsageNotificationIntervalHours', 24)
|
||||
->set('serverDiskUsageNotificationIntervalHours', 0)
|
||||
->call('submit')
|
||||
->assertHasErrors(['serverDiskUsageNotificationIntervalHours'])
|
||||
->set('serverDiskUsageNotificationIntervalHours', 12)
|
||||
->call('submit')
|
||||
->assertHasNoErrors()
|
||||
->assertDispatched('success');
|
||||
|
||||
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(12);
|
||||
});
|
||||
|
||||
it('does not let a member change the notification interval', function () {
|
||||
$team = Team::factory()->create();
|
||||
$user = User::factory()->create();
|
||||
$team->members()->attach($user, ['role' => 'member']);
|
||||
$server = Server::factory()->create(['team_id' => $team->id]);
|
||||
|
||||
$this->actingAs($user);
|
||||
session(['currentTeam' => $team]);
|
||||
|
||||
Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])
|
||||
->set('serverDiskUsageNotificationIntervalHours', 2)
|
||||
->call('submit');
|
||||
|
||||
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
<?php
|
||||
|
||||
use App\Events\ServerReachabilityChanged;
|
||||
use App\Models\NotificationThrottle;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Channels\EmailChannel;
|
||||
@@ -28,7 +29,6 @@ it('sends Unreachable notification when threshold reached and not yet notified',
|
||||
$this->server->settings()->update(['is_reachable' => false]);
|
||||
$this->server->forceFill([
|
||||
'unreachable_count' => 2,
|
||||
'unreachable_notification_sent' => false,
|
||||
])->save();
|
||||
|
||||
ServerReachabilityChanged::dispatch($this->server->fresh());
|
||||
@@ -41,7 +41,6 @@ it('does not send Unreachable on first transient failure (count=1)', function ()
|
||||
$this->server->settings()->update(['is_reachable' => false]);
|
||||
$this->server->forceFill([
|
||||
'unreachable_count' => 1,
|
||||
'unreachable_notification_sent' => false,
|
||||
])->save();
|
||||
|
||||
ServerReachabilityChanged::dispatch($this->server->fresh());
|
||||
@@ -53,8 +52,8 @@ it('does not send Unreachable when already notified', function () {
|
||||
$this->server->settings()->update(['is_reachable' => false]);
|
||||
$this->server->forceFill([
|
||||
'unreachable_count' => 5,
|
||||
'unreachable_notification_sent' => true,
|
||||
])->save();
|
||||
NotificationThrottle::record($this->server, Unreachable::class);
|
||||
|
||||
ServerReachabilityChanged::dispatch($this->server->fresh());
|
||||
|
||||
@@ -65,8 +64,8 @@ it('sends Reachable notification on recovery when previously notified', function
|
||||
$this->server->settings()->update(['is_reachable' => true]);
|
||||
$this->server->forceFill([
|
||||
'unreachable_count' => 0,
|
||||
'unreachable_notification_sent' => true,
|
||||
])->save();
|
||||
NotificationThrottle::record($this->server, Unreachable::class);
|
||||
|
||||
$fresh = $this->server->fresh();
|
||||
expect($fresh->unreachable_notification_sent)->toBeTrue();
|
||||
@@ -82,7 +81,6 @@ it('does not send Reachable when never notified', function () {
|
||||
$this->server->settings()->update(['is_reachable' => true]);
|
||||
$this->server->forceFill([
|
||||
'unreachable_count' => 0,
|
||||
'unreachable_notification_sent' => false,
|
||||
])->save();
|
||||
|
||||
ServerReachabilityChanged::dispatch($this->server->fresh());
|
||||
@@ -94,7 +92,6 @@ it('routes Unreachable notification through EmailChannel when email toggle is on
|
||||
$this->server->settings()->update(['is_reachable' => false]);
|
||||
$this->server->forceFill([
|
||||
'unreachable_count' => 2,
|
||||
'unreachable_notification_sent' => false,
|
||||
])->save();
|
||||
|
||||
ServerReachabilityChanged::dispatch($this->server->fresh());
|
||||
|
||||
@@ -227,3 +227,20 @@ it('rejects an invalid disk usage check frequency without partially updating the
|
||||
->and($server->settings->server_role->value)->toBe('both')
|
||||
->and($server->proxy->get('type'))->toBe('TRAEFIK');
|
||||
});
|
||||
|
||||
it('updates the disk usage notification interval', function () {
|
||||
patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [
|
||||
'server_disk_usage_notification_interval_hours' => 48,
|
||||
])->assertCreated();
|
||||
|
||||
expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(48);
|
||||
});
|
||||
|
||||
it('rejects an out of range disk usage notification interval', function () {
|
||||
patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [
|
||||
'server_disk_usage_notification_interval_hours' => 0,
|
||||
])->assertUnprocessable()
|
||||
->assertJsonValidationErrors('server_disk_usage_notification_interval_hours');
|
||||
|
||||
expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
|
||||
});
|
||||
@@ -42,7 +42,6 @@ test('fix mode fully ends a locally active subscription with an invalid Stripe s
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $this->team->id,
|
||||
'unreachable_count' => 0,
|
||||
'unreachable_notification_sent' => false,
|
||||
]);
|
||||
|
||||
$stripe = Mockery::mock(StripeClient::class);
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\PushServerUpdateJob;
|
||||
use App\Models\Environment;
|
||||
use App\Models\Project;
|
||||
use App\Models\Server;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use App\Notifications\Container\ContainerRestarted;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
Cache::flush();
|
||||
Queue::fake();
|
||||
Notification::fake();
|
||||
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
|
||||
|
||||
$this->team = Team::factory()->create();
|
||||
$this->team->emailNotificationSettings()->update([
|
||||
'use_instance_email_settings' => true,
|
||||
'status_change_email_notifications' => true,
|
||||
]);
|
||||
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
Carbon::setTestNow();
|
||||
});
|
||||
|
||||
it('sends a throttled notification once per interval for the same resource', function () {
|
||||
$otherServer = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
|
||||
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
||||
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
||||
$this->team->notify(new ContainerRestarted('coolify-proxy', $otherServer, restartedResource: $otherServer));
|
||||
|
||||
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
||||
|
||||
Carbon::setTestNow(now()->addMinutes(59));
|
||||
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
||||
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
||||
|
||||
Carbon::setTestNow(now()->addMinutes(2));
|
||||
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
||||
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 3);
|
||||
});
|
||||
|
||||
it('does not throttle a notification without a throttle subject', function () {
|
||||
$this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
|
||||
$this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
|
||||
|
||||
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
||||
});
|
||||
|
||||
it('sends one TCP proxy restart notification per hour while the proxy does not stay up', function () {
|
||||
$project = Project::factory()->create(['team_id' => $this->team->id]);
|
||||
$environment = Environment::factory()->create(['project_id' => $project->id]);
|
||||
$destination = $this->server->standaloneDockers()->firstOrFail();
|
||||
$database = StandalonePostgresql::create([
|
||||
'name' => 'public-db',
|
||||
'postgres_user' => 'postgres',
|
||||
'postgres_password' => encrypt('password'),
|
||||
'postgres_db' => 'app',
|
||||
'image' => 'postgres:16-alpine',
|
||||
'is_public' => true,
|
||||
'public_port' => 54321,
|
||||
'environment_id' => $environment->id,
|
||||
'destination_id' => $destination->id,
|
||||
'destination_type' => $destination->getMorphClass(),
|
||||
]);
|
||||
|
||||
// The database runs, but its "-proxy" container is missing on every push.
|
||||
$push = fn () => (new PushServerUpdateJob($this->server, [
|
||||
'containers' => [[
|
||||
'name' => $database->uuid,
|
||||
'state' => 'running',
|
||||
'health_status' => 'healthy',
|
||||
'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid],
|
||||
]],
|
||||
'filesystem_usage_root' => ['used_percentage' => 10],
|
||||
]))->handle();
|
||||
|
||||
$push();
|
||||
Carbon::setTestNow(now()->addMinute());
|
||||
$push();
|
||||
Carbon::setTestNow(now()->addMinute());
|
||||
$push();
|
||||
|
||||
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 1);
|
||||
|
||||
Carbon::setTestNow(now()->addHour());
|
||||
$push();
|
||||
|
||||
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
||||
});
|
||||
Reference in new issue
Block a user