mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-04 10:42:23 +00:00
Andras Bacsai
99c840922c
High disk usage notifications were sent every ~10 minutes on Coolify Cloud because the cache rate limiter did not hold across Sentinel-triggered checks. - Add a notification_throttles table and NotificationThrottle helper that atomically records when a notification was sent for a resource. - Add the ThrottledNotification contract; Team::notify() skips a throttled notification that was already sent within its interval. - High disk usage: configurable per-server interval (server_disk_usage_notification_interval_hours, default 24h) in Server > Advanced and the servers API. - Container restarted (coolify-proxy, database TCP proxy): at most one notification per hour per server/database during a restart loop. - Move the unreachable and missing-backup markers into the new table and migrate existing data (servers.unreachable_notification_sent and scheduled_database_backups.missing_backup_notification_sent_at). - Regenerate OpenAPI files.
103 lines
4.0 KiB
PHP
103 lines
4.0 KiB
PHP
<?php
|
|
|
|
use App\Jobs\PushServerUpdateJob;
|
|
use App\Models\Environment;
|
|
use App\Models\Project;
|
|
use App\Models\Server;
|
|
use App\Models\StandalonePostgresql;
|
|
use App\Models\Team;
|
|
use App\Notifications\Container\ContainerRestarted;
|
|
use Carbon\Carbon;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Illuminate\Support\Facades\Notification;
|
|
use Illuminate\Support\Facades\Queue;
|
|
|
|
uses(RefreshDatabase::class);
|
|
|
|
beforeEach(function () {
|
|
Server::flushIdentityMap();
|
|
Cache::flush();
|
|
Queue::fake();
|
|
Notification::fake();
|
|
Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
|
|
|
|
$this->team = Team::factory()->create();
|
|
$this->team->emailNotificationSettings()->update([
|
|
'use_instance_email_settings' => true,
|
|
'status_change_email_notifications' => true,
|
|
]);
|
|
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
|
|
});
|
|
|
|
afterEach(function () {
|
|
Carbon::setTestNow();
|
|
});
|
|
|
|
it('sends a throttled notification once per interval for the same resource', function () {
|
|
$otherServer = Server::factory()->create(['team_id' => $this->team->id]);
|
|
|
|
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
|
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
|
$this->team->notify(new ContainerRestarted('coolify-proxy', $otherServer, restartedResource: $otherServer));
|
|
|
|
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
|
|
|
Carbon::setTestNow(now()->addMinutes(59));
|
|
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
|
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
|
|
|
Carbon::setTestNow(now()->addMinutes(2));
|
|
$this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
|
|
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 3);
|
|
});
|
|
|
|
it('does not throttle a notification without a throttle subject', function () {
|
|
$this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
|
|
$this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
|
|
|
|
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
|
});
|
|
|
|
it('sends one TCP proxy restart notification per hour while the proxy does not stay up', function () {
|
|
$project = Project::factory()->create(['team_id' => $this->team->id]);
|
|
$environment = Environment::factory()->create(['project_id' => $project->id]);
|
|
$destination = $this->server->standaloneDockers()->firstOrFail();
|
|
$database = StandalonePostgresql::create([
|
|
'name' => 'public-db',
|
|
'postgres_user' => 'postgres',
|
|
'postgres_password' => encrypt('password'),
|
|
'postgres_db' => 'app',
|
|
'image' => 'postgres:16-alpine',
|
|
'is_public' => true,
|
|
'public_port' => 54321,
|
|
'environment_id' => $environment->id,
|
|
'destination_id' => $destination->id,
|
|
'destination_type' => $destination->getMorphClass(),
|
|
]);
|
|
|
|
// The database runs, but its "-proxy" container is missing on every push.
|
|
$push = fn () => (new PushServerUpdateJob($this->server, [
|
|
'containers' => [[
|
|
'name' => $database->uuid,
|
|
'state' => 'running',
|
|
'health_status' => 'healthy',
|
|
'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid],
|
|
]],
|
|
'filesystem_usage_root' => ['used_percentage' => 10],
|
|
]))->handle();
|
|
|
|
$push();
|
|
Carbon::setTestNow(now()->addMinute());
|
|
$push();
|
|
Carbon::setTestNow(now()->addMinute());
|
|
$push();
|
|
|
|
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 1);
|
|
|
|
Carbon::setTestNow(now()->addHour());
|
|
$push();
|
|
|
|
Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
|
|
});
|