coolify/app/Jobs/PushServerUpdateJob.php

<?php

namespace App\Jobs;

use App\Actions\Application\StopApplication;
use App\Actions\Application\StopApplicationPreview;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Database\StopDatabaseProxy;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\StartLogDrain;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDocker;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Models\StandaloneSqlite;
use App\Models\SwarmDocker;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Server\HighDiskUsage;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Laravel\Horizon\Contracts\Silenced;

class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
{
    use CalculatesExcludedStatus;
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    public $tries = 1;

    public $timeout = 30;

    public Collection $containers;

    public Collection $applications;

    public Collection $previews;

    public Collection $databases;

    public Collection $services;

    public Collection $applicationsById;

    public Collection $previewsByKey;

    public Collection $databasesByUuid;

    public Collection $servicesById;

    public Collection $serviceApplicationsById;

    public Collection $serviceDatabasesById;

    public Collection $allApplicationIds;

    public Collection $allDatabaseUuids;

    public Collection $allTcpProxyUuids;

    public Collection $allServiceApplicationIds;

    public Collection $allApplicationPreviewsIds;

    public Collection $allServiceDatabaseIds;

    public Collection $allApplicationsWithAdditionalServers;

    public Collection $foundApplicationIds;

    public Collection $foundDatabaseUuids;

    public Collection $foundServiceApplicationIds;

    public Collection $foundServiceDatabaseIds;

    public Collection $foundApplicationPreviewsIds;

    public Collection $applicationContainerStatuses;

    public Collection $applicationContainerRestartCounts;

    public Collection $serviceContainerStatuses;

    public Collection $previewContainerRestartCounts;

    public Collection $serviceContainerRestartCounts;

    public bool $foundProxy = false;

    public bool $foundLogDrainContainer = false;

    public array $logDrainContainerLabels = [];

    private ?array $cachedDestinationIds = null;

    public function middleware(): array
    {
        return [(new WithoutOverlapping('push-server-update-'.$this->server->uuid))->expireAfter(30)->dontRelease()];
    }

    public function backoff(): int
    {
        return isDev() ? 1 : 3;
    }

    public function __construct(public Server $server, public $data)
    {
        $this->containers = collect();
        $this->foundApplicationIds = collect();
        $this->foundDatabaseUuids = collect();
        $this->foundServiceApplicationIds = collect();
        $this->foundApplicationPreviewsIds = collect();
        $this->foundServiceDatabaseIds = collect();
        $this->applicationContainerStatuses = collect();
        $this->applicationContainerRestartCounts = collect();
        $this->serviceContainerStatuses = collect();
        $this->previewContainerRestartCounts = collect();
        $this->serviceContainerRestartCounts = collect();
        $this->allApplicationIds = collect();
        $this->allDatabaseUuids = collect();
        $this->allTcpProxyUuids = collect();
        $this->allServiceApplicationIds = collect();
        $this->allServiceDatabaseIds = collect();
        $this->applicationsById = collect();
        $this->previewsByKey = collect();
        $this->databasesByUuid = collect();
        $this->servicesById = collect();
        $this->serviceApplicationsById = collect();
        $this->serviceDatabasesById = collect();
    }

    public function handle()
    {
        // Defensive initialization for Collection properties to handle queue deserialization edge cases
        $this->serviceContainerStatuses ??= collect();
        $this->previewContainerRestartCounts ??= collect();
        $this->serviceContainerRestartCounts ??= collect();
        $this->applicationContainerStatuses ??= collect();
        $this->applicationContainerRestartCounts ??= collect();
        $this->foundApplicationIds ??= collect();
        $this->foundDatabaseUuids ??= collect();
        $this->foundServiceApplicationIds ??= collect();
        $this->foundApplicationPreviewsIds ??= collect();
        $this->foundServiceDatabaseIds ??= collect();
        $this->allApplicationIds ??= collect();
        $this->allDatabaseUuids ??= collect();
        $this->allTcpProxyUuids ??= collect();
        $this->allServiceApplicationIds ??= collect();
        $this->allServiceDatabaseIds ??= collect();
        $this->applicationsById ??= collect();
        $this->previewsByKey ??= collect();
        $this->databasesByUuid ??= collect();
        $this->servicesById ??= collect();
        $this->serviceApplicationsById ??= collect();
        $this->serviceDatabasesById ??= collect();

        // Eager-load relations the job touches repeatedly to avoid lazy-load queries
        // (settings: disk threshold, isProxyShouldRun, isLogDrainEnabled; team: notifications).
        $this->server->loadMissing(['settings', 'team']);

        // TODO: Swarm is not supported yet
        if (! $this->data) {
            throw new \Exception('No data provided');
        }
        $data = collect($this->data);

        // Heartbeat is updated by SentinelController on every push, before dispatch.
        $this->containers = collect(data_get($data, 'containers'));
        $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');

        // Only dispatch the storage check when disk usage is at/above the notification
        // threshold AND the value changed. Below the threshold ServerStorageCheckJob
        // has nothing to do (it only sends a HighDiskUsage notification), so dispatching
        // it is wasted work — and most servers sit well below the threshold.
        $diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
        $storageCacheKey = 'storage-check:'.$this->server->id;
        // Set while usage was high, so the throttle is released only once when usage recovers.
        $highUsageCacheKey = 'storage-high:'.$this->server->id;
        $lastPercentage = Cache::get($storageCacheKey);
        if ($filesystemUsageRoot !== null
            && $filesystemUsageRoot >= $diskThreshold
            && (string) $lastPercentage !== (string) $filesystemUsageRoot) {
            Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
            Cache::forever($highUsageCacheKey, true);
            ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
        } elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) {
            Cache::forget($storageCacheKey);
            // The storage check does not run below the threshold, so the next spike alerts again only
            // when the throttle is released here.
            if (HighDiskUsage::hasRecovered($filesystemUsageRoot, $diskThreshold) && Cache::pull($highUsageCacheKey)) {
                NotificationThrottle::release($this->server, HighDiskUsage::class);
            }
        }

        if ($this->containers->isEmpty() && ! $this->isCompleteSnapshot()) {
            return;
        }

        $this->applications = $this->loadApplications();
        $this->databases = $this->loadDatabases();
        $this->previews = $this->loadPreviews();
        $this->services = $this->loadServices();
        $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
        $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
        $this->databasesByUuid = $this->databases->keyBy('uuid');
        $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
        $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
        $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);

        $this->allApplicationIds = $this->applications->filter(function ($application) {
            return $application->additional_servers_count === 0;
        })->pluck('id');
        $this->allApplicationsWithAdditionalServers = $this->applications->filter(function ($application) {
            return $application->additional_servers_count > 0;
        });
        $this->allApplicationPreviewsIds = $this->previews->map(function ($preview) {
            return $preview->application_id.':'.$preview->pull_request_id;
        });
        $this->allDatabaseUuids = $this->databases->pluck('uuid');
        $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
        $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
        $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();
        $upResourceIds = NotifyResourceStoppedJob::upResourceIds($this->applications->whereIn('id', $this->allApplicationIds)
            ->concat($this->previews)
            ->concat($this->databases)
            ->concat($this->serviceApplicationsById)
            ->concat($this->serviceDatabasesById));

        // Owners of application containers outside this server's applications, in one query.
        $foreignApplicationIdsByUuid = containerApplicationIdsByUuid(
            $this->applications,
            $this->containers
                ->map(fn ($container) => collect(data_get($container, 'labels')))
                ->filter(fn (Collection $labels) => $labels->has('coolify.managed') && isContainerOfType($labels, 'application'))
        );

        foreach ($this->containers as $container) {
            $containerStatus = data_get($container, 'state', 'exited');
            $rawHealthStatus = data_get($container, 'health_status');
            $containerHealth = $rawHealthStatus ?? 'unknown';
            // Only append health status if container is not exited
            if ($containerStatus !== 'exited') {
                $containerStatus = "$containerStatus:$containerHealth";
            }
            $labels = collect(data_get($container, 'labels'));
            $coolify_managed = $labels->has('coolify.managed');

            if (! $coolify_managed) {
                continue;
            }
            if (filter_var($labels->get('com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) {
                continue;
            }

            $name = data_get($container, 'name');
            if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
                $this->foundLogDrainContainer = true;
                $this->logDrainContainerLabels = $labels->all();
            }
            // Containers are matched by owner UUID; numeric ids differ between instances.
            if (isContainerOfType($labels, 'application')) {
                $applicationId = resolveContainerApplicationId($this->applications, $labels, $foreignApplicationIdsByUuid);
                if ($applicationId === null) {
                    continue;
                }
                $applicationId = (string) $applicationId;
                $pullRequestId = $labels->get('coolify.pullRequestId', '0');
                try {
                    if ($pullRequestId === '0') {
                        $application = $this->applicationsById->get((string) $applicationId);
                        if ($application && $application->container_present !== true) {
                            $application->update(['container_present' => true]);
                        }
                        if ($this->allApplicationIds->contains($applicationId)) {
                            $this->foundApplicationIds->push($applicationId);
                        }
                        // Store container status for aggregation
                        if (! $this->applicationContainerStatuses->has($applicationId)) {
                            $this->applicationContainerStatuses->put($applicationId, collect());
                        }
                        $containerName = $labels->get('com.docker.compose.service');
                        if ($containerName) {
                            $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
                            $restartCount = data_get($container, 'restart_count');
                            if (is_numeric($restartCount)) {
                                if (! $this->applicationContainerRestartCounts->has($applicationId)) {
                                    $this->applicationContainerRestartCounts->put($applicationId, collect());
                                }
                                $this->applicationContainerRestartCounts->get($applicationId)->put($containerName, (int) $restartCount);
                            }
                        }
                    } else {
                        $previewKey = $applicationId.':'.$pullRequestId;
                        if ($this->allApplicationPreviewsIds->contains($previewKey)) {
                            $this->foundApplicationPreviewsIds->push($previewKey);
                        }
                        $this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
                        $restartCount = data_get($container, 'restart_count');
                        if (is_numeric($restartCount)) {
                            $this->previewContainerRestartCounts->push([
                                'key' => $previewKey,
                                'count' => (int) $restartCount,
                            ]);
                        }
                    }
                } catch (\Exception $e) {
                }
            } elseif (isContainerOfType($labels, 'service')) {
                [$service, $subType, $servicePart] = resolveServiceContainerOwner($this->services, $labels);
                if (! $service || ! $servicePart) {
                    continue;
                }
                $serviceId = (string) $service->id;
                $subId = (string) $servicePart->id;
                if ($subType === 'application') {
                    $this->foundServiceApplicationIds->push($subId);
                    // Store container status for aggregation
                    $key = $serviceId.':'.$subType.':'.$subId;
                    if (! $this->serviceContainerStatuses->has($key)) {
                        $this->serviceContainerStatuses->put($key, collect());
                    }
                    $containerName = $labels->get('com.docker.compose.service');
                    if ($containerName) {
                        $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
                        $this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count'));
                    }
                } elseif ($subType === 'database') {
                    $this->foundServiceDatabaseIds->push($subId);
                    // Store container status for aggregation
                    $key = $serviceId.':'.$subType.':'.$subId;
                    if (! $this->serviceContainerStatuses->has($key)) {
                        $this->serviceContainerStatuses->put($key, collect());
                    }
                    $containerName = $labels->get('com.docker.compose.service');
                    if ($containerName) {
                        $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
                        $this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count'));
                    }
                }
            } else {
                $uuid = $labels->get('com.docker.compose.service');
                $type = $labels->get('coolify.type');
                if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
                    $this->foundProxy = true;
                } elseif ($type === 'service' && $this->isRunning($containerStatus)) {
                } else {
                    if ($this->allDatabaseUuids->contains($uuid) && $this->isActiveOrTransient($containerStatus)) {
                        $this->foundDatabaseUuids->push($uuid);
                        // TCP proxy should only be started/managed when database is actually running
                        if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
                            $this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: true);
                        } else {
                            $this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: false);
                        }
                    }
                }
            }
        }

        if (! $this->isCompleteSnapshot()) {
            ServiceChecked::dispatch($this->server->team_id);

            return;
        }

        $this->updateProxyStatus();

        Application::whereIn('id', $this->foundApplicationIds->unique())
            ->update(['container_present' => true]);

        $this->updateNotFoundApplicationStatus();
        $this->updateNotFoundApplicationPreviewStatus();
        $this->updateNotFoundDatabaseStatus();
        $this->updateNotFoundServiceStatus();

        $this->updateAdditionalServersStatus();

        $this->trackPreviewRestartCounts();

        // Aggregate multi-container application statuses
        $this->aggregateMultiContainerStatuses();

        // Aggregate multi-container service statuses
        $this->aggregateServiceContainerStatuses();

        NotifyResourceStoppedJob::dispatchForStopped($upResourceIds);

        $this->checkLogDrainContainer();

        ServiceChecked::dispatch($this->server->team_id);
    }

    private function isCompleteSnapshot(): bool
    {
        return data_get($this->data, 'snapshot.complete', true) !== false;
    }

    private function loadApplications(): Collection
    {
        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();

        $applications = ($standaloneDockerIds->isNotEmpty() || $swarmDockerIds->isNotEmpty())
            ? Application::withoutGlobalScope('withRelations')
                ->select([
                    'id',
                    'uuid',
                    'name',
                    'status',
                    'container_present',
                    'build_pack',
                    'docker_compose_raw',
                    'environment_id',
                    'destination_id',
                    'destination_type',
                    'last_online_at',
                    'restart_count',
                    'max_restart_count',
                    'restart_limit_reached',
                    'last_restart_at',
                    'last_restart_type',
                ])
                ->withCount('additional_servers')
                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
                ->get()
            : collect();

        $additionalApplicationIds = DB::table('additional_destinations')
            ->where('server_id', $this->server->id)
            ->pluck('application_id');

        if ($additionalApplicationIds->isNotEmpty()) {
            $applications = $applications->concat(
                Application::withoutGlobalScope('withRelations')
                    ->select([
                        'id',
                        'uuid',
                        'name',
                        'status',
                        'container_present',
                        'build_pack',
                        'docker_compose_raw',
                        'environment_id',
                        'destination_id',
                        'destination_type',
                        'last_online_at',
                        'restart_count',
                        'max_restart_count',
                        'restart_limit_reached',
                        'last_restart_at',
                        'last_restart_type',
                    ])
                    ->withCount('additional_servers')
                    ->whereIn('id', $additionalApplicationIds)
                    ->get()
            );
        }

        return $applications->unique('id')->values();
    }

    private function loadPreviews(): Collection
    {
        $applicationIds = $this->applications->pluck('id');

        if ($applicationIds->isEmpty()) {
            return collect();
        }

        return ApplicationPreview::query()
            ->select([
                'id',
                'application_id',
                'pull_request_id',
                'status',
                'last_online_at',
                'restart_count',
                'max_restart_count',
                'restart_limit_reached',
                'last_restart_at',
                'last_restart_type',
            ])
            ->whereIn('application_id', $applicationIds)
            ->get();
    }

    private function loadServices(): Collection
    {
        return $this->server->services()
            ->select([
                'id',
                'server_id',
                'uuid',
                'docker_compose_raw',
            ])
            ->with([
                'applications:id,uuid,name,service_id,status,last_online_at,restart_count,max_restart_count,restart_limit_reached,last_restart_at,last_restart_type',
                'databases:id,uuid,name,service_id,status,last_online_at,is_public',
            ])
            ->get();
    }

    private function loadDatabases(): Collection
    {
        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
        if ($standaloneDockerIds->isEmpty() && $swarmDockerIds->isEmpty()) {
            return collect();
        }
        $databaseColumns = [
            'id',
            'uuid',
            'name',
            'status',
            'is_public',
            'destination_id',
            'destination_type',
            'last_online_at',
            'restart_count',
            'last_restart_at',
            'last_restart_type',
        ];

        return collect([
            StandalonePostgresql::class,
            StandaloneRedis::class,
            StandaloneMongodb::class,
            StandaloneMysql::class,
            StandaloneMariadb::class,
            StandaloneKeydb::class,
            StandaloneDragonfly::class,
            StandaloneClickhouse::class,
            StandaloneSqlite::class,
        ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
            return $databaseClass::query()
                ->select($databaseColumns)
                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
                ->get();
        })->reject(fn ($database) => $database instanceof StandalonePostgresql && $database->isInstanceDatabase())->values();
    }

    private function serverDestinationIds(): array
    {
        if ($this->cachedDestinationIds !== null) {
            return $this->cachedDestinationIds;
        }

        return $this->cachedDestinationIds = [
            StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
            SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
        ];
    }

    private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
    {
        $query->where(function ($query) use ($standaloneDockerIds) {
            $query->where('destination_type', StandaloneDocker::class)
                ->whereIn('destination_id', $standaloneDockerIds);
        })->orWhere(function ($query) use ($swarmDockerIds) {
            $query->where('destination_type', SwarmDocker::class)
                ->whereIn('destination_id', $swarmDockerIds);
        });
    }

    private function aggregateMultiContainerStatuses()
    {
        if ($this->applicationContainerStatuses->isEmpty()) {
            return;
        }

        foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
            $application = $this->applicationsById->get((string) $applicationId);
            if (! $application) {
                continue;
            }

            $maxRestartCount = 0;
            $restartCountsAvailable = $this->applicationContainerRestartCounts->has($applicationId);
            if ($restartCountsAvailable) {
                $maxRestartCount = $this->applicationContainerRestartCounts->get($applicationId)->max() ?? 0;
                $restartState = (new RestartCountTracker)->evaluate(
                    previousRestartCount: $application->restart_count ?? 0,
                    observedRestartCount: $maxRestartCount,
                    maxRestartCount: $application->max_restart_count ?? 0,
                );

                if ($restartState['restart_count_changed']) {
                    $hasCrashRestarts = $restartState['restart_count'] > 0;
                    $application->update([
                        'restart_count' => $restartState['restart_count'],
                        'last_restart_at' => $hasCrashRestarts ? now() : null,
                        'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
                    ]);
                }

                if ($restartState['restart_limit_reached']) {
                    $restartLimitClaimed = Application::query()
                        ->whereKey($application->getKey())
                        ->where('restart_limit_reached', false)
                        ->update(['restart_limit_reached' => true]) === 1;

                    // Stop again while a container still runs: an earlier stop can fail or come too late.
                    $stillRunning = $containerStatuses->contains(fn (string $status): bool => ! str($status)->contains('exited'));
                    if ($restartLimitClaimed || $stillRunning) {
                        $application->refresh();
                        StopApplication::dispatch(
                            application: $application,
                            previewDeployments: false,
                            dockerCleanup: false,
                            resetRestartCount: false,
                            removeContainers: false,
                        );
                    }
                    if ($restartLimitClaimed) {
                        $application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
                    }
                }
            }

            if ($application->stoppedAfterRestartLimit() && $containerStatuses->every(
                fn (string $status): bool => str($status)->contains('exited')
            )) {
                $application->update(['status' => 'exited']);

                continue;
            }

            // Parse docker compose to check for excluded containers
            $dockerComposeRaw = data_get($application, 'docker_compose_raw');
            $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);

            // Filter out excluded containers
            $relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
                return ! $excludedContainers->contains($containerName);
            });

            // If all containers are excluded, calculate status from excluded containers
            if ($relevantStatuses->isEmpty()) {
                $aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);

                if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                    $application->status = $aggregatedStatus;
                    $application->save();
                }

                continue;
            }

            // Use ContainerStatusAggregator service for state machine logic
            // Use preserveRestarting: true so applications show "Restarting" instead of "Degraded"
            $aggregator = new ContainerStatusAggregator;
            $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount, preserveRestarting: true);

            // Update application status with aggregated result
            if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                $application->status = $aggregatedStatus;
                $application->save();
            }
        }
    }

    private function aggregateServiceContainerStatuses()
    {
        if ($this->serviceContainerStatuses->isEmpty()) {
            return;
        }

        foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
            // Parse key: serviceId:subType:subId
            [$serviceId, $subType, $subId] = explode(':', $key);

            if (empty($subId)) {
                continue;
            }

            $service = $this->servicesById->get((string) $serviceId);
            if (! $service) {
                continue;
            }

            // Get the service sub-resource (ServiceApplication or ServiceDatabase)
            $subResource = null;
            if ($subType === 'application') {
                $subResource = $this->serviceApplicationsById->get((string) $subId);
            } elseif ($subType === 'database') {
                $subResource = $this->serviceDatabasesById->get((string) $subId);
            }

            if (! $subResource) {
                continue;
            }

            $restartCount = $this->serviceContainerRestartCounts->get($key)?->max() ?? 0;
            if (! $subResource instanceof ServiceDatabase && $subResource->trackRestartCount($restartCount)) {
                StopServiceApplication::dispatch($subResource, false, false);
                $subResource->team()?->notify(new ApplicationRestartLimitReached($subResource));

                continue;
            }

            // Parse docker compose from service to check for excluded containers
            $dockerComposeRaw = data_get($service, 'docker_compose_raw');
            $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);

            // Filter out excluded containers
            $relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
                return ! $excludedContainers->contains($containerName);
            });

            // If all containers are excluded, calculate status from excluded containers
            if ($relevantStatuses->isEmpty()) {
                $aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
                if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                    $subResource->status = $aggregatedStatus;
                    $subResource->save();
                }

                continue;
            }

            // Use ContainerStatusAggregator service for state machine logic
            // Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded"
            $aggregator = new ContainerStatusAggregator;
            $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $restartCount, preserveRestarting: true);

            // Update service sub-resource status with aggregated result
            if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                $subResource->status = $aggregatedStatus;
                $subResource->save();
            }
        }
    }

    private function updateApplicationStatus(string $applicationId, string $containerStatus)
    {
        $application = $this->applicationsById->get((string) $applicationId);
        if (! $application) {
            return;
        }
        if ($application->status !== $containerStatus) {
            $application->status = $containerStatus;
            $application->save();
        }
    }

    private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
    {
        $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
        if (! $application) {
            return;
        }
        if ($application->status !== $containerStatus) {
            $application->status = $containerStatus;
            $application->save();
        }
    }

    private function updateNotFoundApplicationStatus()
    {
        $notFoundApplicationIds = $this->allApplicationIds->diff($this->foundApplicationIds);
        if ($notFoundApplicationIds->isEmpty()) {
            return;
        }

        // Batch update: mark all not-found applications as exited (excluding already exited ones)
        Application::whereIn('id', $notFoundApplicationIds)
            ->update([
                'status' => 'exited',
                'container_present' => false,
                'restart_limit_reached' => false,
            ]);
    }

    private function updateNotFoundApplicationPreviewStatus()
    {
        $notFoundApplicationPreviewsIds = $this->allApplicationPreviewsIds->diff($this->foundApplicationPreviewsIds);
        if ($notFoundApplicationPreviewsIds->isEmpty()) {
            return;
        }

        // Collect IDs of previews that need to be marked as exited
        $previewIdsToUpdate = collect();
        foreach ($notFoundApplicationPreviewsIds as $previewKey) {
            // Parse the previewKey format "application_id:pull_request_id"
            $parts = explode(':', $previewKey);
            if (count($parts) !== 2) {
                continue;
            }

            $applicationId = $parts[0];
            $pullRequestId = $parts[1];

            $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);

            if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
                $previewIdsToUpdate->push($applicationPreview->id);
            }
        }

        // Batch update all collected preview IDs
        if ($previewIdsToUpdate->isNotEmpty()) {
            ApplicationPreview::whereIn('id', $previewIdsToUpdate)->update(['status' => 'exited']);
        }
    }

    private function updateProxyStatus()
    {
        // If proxy is not found, start it
        if ($this->server->isProxyShouldRun() && ! $this->server->isProxyTransitionInProgress()) {
            if ($this->foundProxy === false) {
                try {
                    if (CheckProxy::run($this->server)) {
                        StartProxy::run($this->server, async: false);
                        $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
                    }
                } catch (\Throwable $e) {
                }
            } elseif ($this->server->proxy->get('status') !== 'running') {
                $this->server->proxy->set('status', 'running');
                $this->server->save();
            }
        }
    }

    private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, mixed $restartCount = null, bool $tcpProxy = false): void
    {
        $database = $this->databasesByUuid->get($databaseUuid);
        if (! $database) {
            return;
        }
        if ($database->status !== $containerStatus) {
            $database->status = $containerStatus;
            $database->save();
        }
        if (is_numeric($restartCount) && $restartCount > ($database->restart_count ?? 0)) {
            $database->update([
                'restart_count' => (int) $restartCount,
                'last_restart_at' => now(),
                'last_restart_type' => 'crash',
            ]);
        }
        if (! $this->isCompleteSnapshot()) {
            return;
        }
        if ($this->isRunning($containerStatus) && $tcpProxy) {
            $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
                return data_get($value, 'name') === "$databaseUuid-proxy" && data_get($value, 'state') === 'running';
            })->first();
            if (! $tcpProxyContainerFound) {
                StartDatabaseProxy::dispatch($database);
                $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database));
            } elseif (StartDatabaseProxy::claimOutdatedRefresh($databaseUuid, data_get($tcpProxyContainerFound, 'labels'))) {
                StartDatabaseProxy::dispatch($database);
            }
        } elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
            // Clean up orphaned proxy containers when is_public=false
            $orphanedProxy = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
                return data_get($value, 'name') === "$databaseUuid-proxy" && data_get($value, 'state') === 'running';
            })->first();
            if ($orphanedProxy) {
                StopDatabaseProxy::dispatch($database);
            }
        }
    }

    private function storeServiceRestartCount(string $key, string $containerName, mixed $restartCount): void
    {
        if (! is_numeric($restartCount)) {
            return;
        }
        if (! $this->serviceContainerRestartCounts->has($key)) {
            $this->serviceContainerRestartCounts->put($key, collect());
        }
        $this->serviceContainerRestartCounts->get($key)->put($containerName, (int) $restartCount);
    }

    private function trackPreviewRestartCounts(): void
    {
        $this->previewContainerRestartCounts
            ->groupBy('key')
            ->each(function (Collection $counts, string $key): void {
                $preview = $this->previewsByKey->get($key);
                if ($preview?->trackRestartCount((int) $counts->max('count'))) {
                    StopApplicationPreview::dispatch($preview, false, false);
                    $preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview));
                }
            });
    }

    private function updateNotFoundDatabaseStatus()
    {
        $notFoundDatabaseUuids = $this->allDatabaseUuids->diff($this->foundDatabaseUuids);
        if ($notFoundDatabaseUuids->isEmpty()) {
            return;
        }

        $notFoundDatabaseUuids->each(function ($databaseUuid) {
            $database = $this->databasesByUuid->get($databaseUuid);
            if ($database) {
                if (! str($database->status)->startsWith('exited')) {
                    $database->update([
                        'status' => 'exited',
                        'restart_count' => 0,
                        'last_restart_at' => null,
                        'last_restart_type' => null,
                    ]);
                }
                if ($database->is_public) {
                    StopDatabaseProxy::dispatch($database);
                }
            }
        });
    }

    private function updateNotFoundServiceStatus()
    {
        $notFoundServiceApplicationIds = $this->allServiceApplicationIds->diff($this->foundServiceApplicationIds);
        $notFoundServiceDatabaseIds = $this->allServiceDatabaseIds->diff($this->foundServiceDatabaseIds);

        // Batch update service applications
        if ($notFoundServiceApplicationIds->isNotEmpty()) {
            ServiceApplication::whereIn('id', $notFoundServiceApplicationIds)
                ->where('restart_limit_reached', false)
                ->where('status', '!=', 'exited')
                ->update(['status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null]);
        }

        // Batch update service databases
        if ($notFoundServiceDatabaseIds->isNotEmpty()) {
            ServiceDatabase::whereIn('id', $notFoundServiceDatabaseIds)
                ->where('status', '!=', 'exited')
                ->update(['status' => 'exited']);
        }
    }

    private function updateAdditionalServersStatus()
    {
        $this->allApplicationsWithAdditionalServers->each(function ($application) {
            ComplexStatusCheck::run($application);
        });
    }

    private function isRunning(string $containerStatus)
    {
        return str($containerStatus)->contains('running');
    }

    /**
     * Check if container is in an active or transient state.
     * Active states: running
     * Transient states: restarting, starting, created, paused
     *
     * These states indicate the container exists and should be tracked.
     * Terminal states (exited, dead, removing) should NOT be tracked.
     */
    private function isActiveOrTransient(string $containerStatus): bool
    {
        return str($containerStatus)->contains('running') ||
               str($containerStatus)->contains('restarting') ||
               str($containerStatus)->contains('starting') ||
               str($containerStatus)->contains('created') ||
               str($containerStatus)->contains('paused');
    }

    private function checkLogDrainContainer()
    {
        if (! $this->server->isFluentBitLogDrainEnabled()) {
            return;
        }
        if ($this->foundLogDrainContainer === false || StartLogDrain::claimOutdatedRefresh($this->server, $this->logDrainContainerLabels)) {
            StartLogDrain::dispatch($this->server);
        }
    }
}