fix(metrics): fetch fleet data concurrently and refresh server access

Run metric requests in batches of eight workers, isolate endpoint failures,
and preserve CPU and memory history when Sentinel lacks a summary endpoint.
Repair Sentinel tokens before creating workers and bound SSH fetch timeouts.

Reauthorize team servers on refresh and hydration, clear stale metrics after
transfers or deletions, and resolve container metadata for team ID zero.

Use only root filesystem samples when mount fields are present, while
supporting legacy disk data. Add regression coverage for concurrent workers,
access changes, endpoint failures, and disk mapping.

.ai/lessons.md

@@ -4,6 +4,7 @@
 - Reproduce the reported failure on the unchanged baseline before adding a fix.
 - When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround.
 - Test old reports against the current branch because later changes can make the report obsolete.
+- Test serialized worker tasks with the real process driver. An in-process test does not prove that captured models or nested closures survive serialization.
 - Use the same regression test before and after the production change so the result shows the behavior difference.
 
 ## Verify the complete user flow

app/Livewire/Fleet/Metrics.php

@@ -8,7 +8,9 @@ use App\Models\Server;
 use App\Services\FleetMetricsAggregator;
 use App\Services\SentinelMetricsClient;
 use Illuminate\Contracts\View\View;
+use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Concurrency;
 use Livewire\Attributes\Lazy;
 use Livewire\Attributes\Url;
 use Livewire\Component;
@@ -16,6 +18,7 @@ use Livewire\Component;
 #[Lazy]
 class Metrics extends Component
 {
+    use AuthorizesRequests;
     use BuildsMetricsChartPayload;
 
     public string $chartId = 'fleet-metrics';
@@ -59,12 +62,39 @@ class Metrics extends Component
 
     public function mount(): void
     {
-        $allServers = Server::ownedByCurrentTeamCached();
+        $this->refreshServers();
 
-        $this->servers = $allServers
+        $this->normalizeContainerMetric();
+
+        if ($this->servers->isNotEmpty()) {
+            $this->loadData();
+        }
+    }
+
+    public function hydrate(): void
+    {
+        $previous = array_column($this->serverRows, 'uuid');
+        $this->refreshServers();
+        $current = $this->targetServers()->pluck('uuid')->all();
+        sort($previous);
+        sort($current);
+
+        if ($previous !== $current) {
+            $this->loadData();
+        }
+    }
+
+    private function refreshServers(): void
+    {
+        abort_unless(currentTeam() !== null, 403);
+        $this->servers = Server::ownedByCurrentTeam()->get()
             ->filter(fn (Server $server) => $server->isMetricsEnabled())
             ->values();
 
+        foreach ($this->servers as $server) {
+            $this->authorize('view', $server);
+        }
+
         $this->serverOptions = $this->servers
             ->mapWithKeys(fn (Server $server) => [$server->uuid => $server->name])
             ->all();
@@ -72,12 +102,6 @@ class Metrics extends Component
         if ($this->serverUuid !== '' && ! array_key_exists($this->serverUuid, $this->serverOptions)) {
             $this->serverUuid = '';
         }
-
-        $this->normalizeContainerMetric();
-
-        if ($this->servers->isNotEmpty()) {
-            $this->loadData();
-        }
     }
 
     public function updatedServerUuid(): void
@@ -125,21 +149,39 @@ class Metrics extends Component
 
     public function loadData(): void
     {
-        if ($this->servers->isEmpty()) {
-            return;
-        }
+        $this->refreshServers();
+        $this->normalizeContainerMetric();
 
         $rows = [];
         $containers = [];
         $series = ['cpu' => [], 'memory' => [], 'disk' => [], 'load' => [], 'networkRx' => [], 'networkTx' => []];
 
-        foreach ($this->targetServers() as $server) {
-            $client = $this->metricsClient($server);
+        $targets = $this->targetServers();
+        $clients = [];
+        $summaryTasks = [];
+        foreach ($targets as $server) {
+            $client = $clients[$server->uuid] = $this->metricsClient($server);
+            $summaryTasks[$server->uuid] = static fn () => $client->summary();
+        }
+        $summaries = $this->runMetricTasks($summaryTasks);
+        $tasks = [];
+        $range = $this->range;
+        foreach ($clients as $uuid => $client) {
+            $metrics = $summaries[$uuid] === null ? ['cpu', 'memory'] : ['cpu', 'memory', 'disk', 'load'];
+            foreach ($metrics as $metric) {
+                $tasks[$uuid.':'.$metric] = static fn () => $client->history($metric, $range);
+            }
+            if ($summaries[$uuid] !== null) {
+                $tasks[$uuid.':containers'] = static fn () => $client->containersCurrent();
+                $tasks[$uuid.':network'] = static fn () => $client->networkHistory($range);
+            }
+        }
+        $results = $this->runMetricTasks($tasks);
 
-            try {
-                $summary = $client->summary();
-            } catch (\Throwable) {
-                $summary = null;
+        foreach ($targets as $server) {
+            $summary = $summaries[$server->uuid];
+            foreach (['cpu', 'memory', 'disk', 'load'] as $metric) {
+                $series[$metric][] = $results[$server->uuid.':'.$metric] ?? [];
             }
 
             if ($summary === null) {
@@ -148,7 +190,7 @@ class Metrics extends Component
                 continue;
             }
 
-            $current = $client->containersCurrent();
+            $current = $results[$server->uuid.':containers'] ?? [];
             $rows[] = array_merge(
                 ['uuid' => $server->uuid, 'name' => $server->name, 'online' => true, 'containers' => count($current)],
                 $summary
@@ -160,11 +202,7 @@ class Metrics extends Component
                 $containers[] = array_merge($c, ['server' => $server->name, 'serverUuid' => $server->uuid]);
             }
 
-            $series['cpu'][] = $client->history('cpu', $this->range);
-            $series['memory'][] = $client->history('memory', $this->range);
-            $series['disk'][] = $client->history('disk', $this->range);
-            $series['load'][] = $client->history('load', $this->range);
-            $net = $client->networkHistory($this->range);
+            $net = $results[$server->uuid.':network'] ?? ['rx' => [], 'tx' => []];
             $series['networkRx'][] = $net['rx'];
             $series['networkTx'][] = $net['tx'];
         }
@@ -178,6 +216,31 @@ class Metrics extends Component
         $this->dispatch("refreshChartData-{$this->chartId}", $this->chartData);
     }
 
+    /**
+     * Bound the number of PHP workers and SSH connections for large fleets.
+     *
+     * @param  array<string, \Closure>  $tasks
+     * @return array<string, mixed>
+     */
+    protected function runMetricTasks(array $tasks): array
+    {
+        foreach ($tasks as $key => $task) {
+            $tasks[$key] = static function () use ($task) {
+                try {
+                    return $task();
+                } catch (\Throwable) {
+                    return null;
+                }
+            };
+        }
+        $results = [];
+        foreach (array_chunk($tasks, 8, true) as $batch) {
+            $results += Concurrency::run($batch);
+        }
+
+        return $results;
+    }
+
     /**
      * @return array<string, mixed>
      */
@@ -193,6 +256,8 @@ class Metrics extends Component
 
     protected function metricsClient(Server $server): SentinelMetricsClient
     {
+        $server->settings->ensureValidSentinelToken();
+
         return app(SentinelMetricsClient::class, ['server' => $server]);
     }
 
@@ -211,7 +276,7 @@ class Metrics extends Component
         }
 
         $teamId = currentTeam()?->id;
-        $resource = $teamId ? getResourceByUuid($id, $teamId) : null;
+        $resource = $teamId !== null ? getResourceByUuid($id, $teamId) : null;
 
         $image = null;
         $link = null;

app/Services/SentinelMetricsClient.php

@@ -108,7 +108,7 @@ class SentinelMetricsClient
             default => ['/cpu/history', 'percent'],
         };
 
-        return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range);
+        return $this->seriesRows($this->url($path, ['from' => self::rangeFrom($range)]), $field, $range, $metric === 'disk');
     }
 
     /**
@@ -127,7 +127,7 @@ class SentinelMetricsClient
     /**
      * @return array<int, array{0: int, 1: float}>
      */
-    private function seriesRows(string $url, string $field, string $range): array
+    private function seriesRows(string $url, string $field, string $range, bool $rootOnly = false): array
     {
         try {
             $rows = json_decode($this->raw($url), true);
@@ -140,9 +140,8 @@ class SentinelMetricsClient
         }
 
         // Disk history interleaves every mount; keep the root filesystem only.
-        $root = array_filter($rows, fn ($r) => ($r['mount'] ?? null) === '/');
-        if ($root !== []) {
-            $rows = $root;
+        if ($rootOnly && array_any($rows, fn ($row) => array_key_exists('mount', $row))) {
+            $rows = array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/');
         }
 
         // Average raw samples into epoch-aligned buckets so every server shares the same
@@ -166,11 +165,14 @@ class SentinelMetricsClient
      */
     private function rootMount(array $mounts): array
     {
-        if ($mounts === []) {
-            return ['used' => null, 'total' => null, 'percent' => null];
+        $root = collect($mounts)->firstWhere('mount', '/');
+        if ($root === null && ! array_any($mounts, fn ($mount) => array_key_exists('mount', $mount))) {
+            $root = $mounts[0] ?? null;
         }
 
-        $root = collect($mounts)->firstWhere('mount', '/') ?? $mounts[0];
+        if ($root === null) {
+            return ['used' => null, 'total' => null, 'percent' => null];
+        }
 
         return [
             'used' => (int) ($root['used'] ?? 0),
@@ -206,7 +208,8 @@ class SentinelMetricsClient
         return instant_remote_process(
             ["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"],
             $this->server,
-            false
+            false,
+            timeout: 15
         );
     }
 

app/Traits/HasMetrics.php

@@ -38,9 +38,11 @@ trait HasMetrics
             return null;
         }
 
-        $root = collect($rows)->filter(fn ($r) => ($r['mount'] ?? null) === '/')->values()->all();
+        if (array_any($rows, fn ($row) => array_key_exists('mount', $row))) {
+            $rows = array_values(array_filter($rows, fn ($row) => ($row['mount'] ?? null) === '/'));
+        }
 
-        return $this->mapRows($root ?: $rows, 'usedPercent', $mins);
+        return $this->mapRows($rows, 'usedPercent', $mins);
     }
 
     /**

config/app.php

@@ -11,6 +11,7 @@ use Illuminate\Auth\Passwords\PasswordResetServiceProvider;
 use Illuminate\Broadcasting\BroadcastServiceProvider;
 use Illuminate\Bus\BusServiceProvider;
 use Illuminate\Cache\CacheServiceProvider;
+use Illuminate\Concurrency\ConcurrencyServiceProvider;
 use Illuminate\Cookie\CookieServiceProvider;
 use Illuminate\Database\DatabaseServiceProvider;
 use Illuminate\Encryption\EncryptionServiceProvider;
@@ -217,6 +218,7 @@ return [
         BroadcastServiceProvider::class,
         BusServiceProvider::class,
         CacheServiceProvider::class,
+        ConcurrencyServiceProvider::class,
         ConsoleSupportServiceProvider::class,
         CookieServiceProvider::class,
         DatabaseServiceProvider::class,

resources/views/livewire/fleet/metrics.blade.php

@@ -29,7 +29,8 @@ $serverListboxOptions = array_merge(
             <div class="flex flex-wrap items-center gap-2">
                 <div class="relative w-full transition-opacity sm:w-52"
                     wire:loading.class="pointer-events-none opacity-60" wire:target="serverUuid">
-                    <x-forms.listbox id="serverUuid" live :options="$serverListboxOptions" placeholder="All servers" />
+                    <x-forms.listbox id="serverUuid" live :options="$serverListboxOptions" placeholder="All servers"
+                        canGate="view" :canResource="$servers->first()" />
                     <div class="absolute inset-0 hidden items-center justify-center rounded-lg bg-white/70 dark:bg-base/70"
                         wire:loading.flex wire:target="serverUuid">
                         <x-loading compact aria-label="Loading metrics" />

tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php

@@ -8,6 +8,7 @@ use App\Models\Team;
 use App\Models\User;
 use App\Services\SentinelMetricsClient;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Concurrency;
 use Livewire\Livewire;
 
 uses(RefreshDatabase::class);
@@ -62,6 +63,7 @@ class DenseTestableMetrics extends Metrics
 }
 
 beforeEach(function () {
+    Concurrency::setDefaultInstance('sync');
     InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
     $user = User::factory()->create();
     $team = Team::factory()->create();

tests/Feature/Livewire/Fleet/MetricsDataTest.php

@@ -1,14 +1,20 @@
 <?php
 
 use App\Livewire\Fleet\Metrics;
+use App\Models\Application;
+use App\Models\Environment;
 use App\Models\InstanceSettings;
 use App\Models\PrivateKey;
+use App\Models\Project;
 use App\Models\Server;
+use App\Models\ServerSetting;
 use App\Models\Team;
 use App\Models\User;
 use App\Services\SentinelMetricsClient;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Concurrency;
 use Livewire\Livewire;
+use Tests\Support\ConcurrentMetricsClient;
 
 uses(RefreshDatabase::class);
 
@@ -20,6 +26,10 @@ class StubMetricsClient extends SentinelMetricsClient
     /** @var array<string, array> serverUuid => containers */
     public static array $containers = [];
 
+    public static array $histories = [];
+
+    public static array $failingMetrics = [];
+
     public function summary(): ?array
     {
         return self::$summaries[$this->server->uuid] ?? null;
@@ -32,7 +42,11 @@ class StubMetricsClient extends SentinelMetricsClient
 
     public function history(string $metric, string $range): array
     {
-        return [];
+        if (in_array($metric, self::$failingMetrics, true)) {
+            throw new RuntimeException('metric unavailable');
+        }
+
+        return self::$histories[$this->server->uuid][$metric] ?? [];
     }
 
     public function networkHistory(string $range): array
@@ -49,9 +63,20 @@ class TestableMetrics extends Metrics
     }
 }
 
+class ProcessTestableMetrics extends Metrics
+{
+    protected function metricsClient(Server $server): SentinelMetricsClient
+    {
+        return new ConcurrentMetricsClient($server);
+    }
+}
+
 beforeEach(function () {
+    Concurrency::setDefaultInstance('sync');
     StubMetricsClient::$summaries = [];
     StubMetricsClient::$containers = [];
+    StubMetricsClient::$histories = [];
+    StubMetricsClient::$failingMetrics = [];
 
     InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
     $user = User::factory()->create();
@@ -227,3 +252,131 @@ it('passes the spark time axis to the KPI sparklines on first paint', function (
 
     $component->assertSeeHtml('1700000100000');
 });
+
+it('removes a transferred server on refresh', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+    $component = Livewire::test(TestableMetrics::class);
+    expect($component->get('serverRows'))->toHaveCount(1);
+
+    $server->update(['team_id' => Team::factory()->create()->id]);
+    $component->call('loadData');
+
+    expect($component->get('serverRows'))->toBe([]);
+    expect($component->get('containersRaw'))->toBe([]);
+    expect($component->get('kpis')['serversTotal'])->toBe(0);
+});
+
+it('permits team members to read metrics', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+    $member = User::factory()->create();
+    $member->teams()->attach($this->team, ['role' => 'member']);
+    $this->actingAs($member);
+
+    expect(Livewire::test(TestableMetrics::class)->get('kpis')['serversOnline'])->toBe(1);
+});
+
+it('keeps host trends when an older Sentinel has no summary endpoint', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]], 'memory' => [[1000, 100.0]]]];
+
+    $component = Livewire::test(TestableMetrics::class);
+
+    expect($component->get('chartData')['cpu'])->toBe([[1000, 25.0]]);
+    expect($component->get('chartData')['memory'])->toBe([[1000, 100.0]]);
+    expect($component->get('topContainers'))->toBe([]);
+});
+
+it('keeps other metrics when one independent operation fails', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+    StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]]]];
+    StubMetricsClient::$failingMetrics = ['disk'];
+
+    $component = Livewire::test(TestableMetrics::class);
+
+    expect($component->get('chartData')['cpu'])->toBe([[1000, 25.0]]);
+    expect($component->get('chartData')['disk'])->toBe([]);
+    expect($component->get('kpis')['serversOnline'])->toBe(1);
+});
+
+it('resolves container names for the instance team with id zero', function () {
+    $rootTeam = Team::factory()->create(['id' => 0]);
+    auth()->user()->teams()->attach($rootTeam, ['role' => 'owner']);
+    auth()->user()->unsetRelation('teams');
+    session(['currentTeam' => $rootTeam]);
+    $project = Project::factory()->create(['team_id' => 0]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+    $application = Application::factory()->create(['environment_id' => $environment->id]);
+
+    $meta = Livewire::test(TestableMetrics::class)->instance()->containerMeta($application->uuid);
+
+    expect($meta['name'])->toBe($application->name);
+    expect($meta['link'])->toContain($application->uuid);
+});
+
+it('denies metrics refresh after team membership is removed', function () {
+    $this->withoutVite();
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+    $component = Livewire::test(TestableMetrics::class);
+
+    auth()->user()->teams()->detach($this->team);
+    auth()->user()->unsetRelation('teams');
+
+    $component->call('loadData')->assertForbidden();
+});
+
+it('clears stale container rankings after a server transfer', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+    StubMetricsClient::$containers = [$server->uuid => [['id' => 'private-container', 'cpu' => 10]]];
+    $component = Livewire::test(TestableMetrics::class);
+
+    $server->update(['team_id' => Team::factory()->create()->id]);
+    $component->set('containerMetric', 'memory');
+
+    expect($component->get('topContainers'))->toBe([]);
+    expect($component->get('serverRows'))->toBe([]);
+});
+
+it('merges metrics from real concurrent workers and isolates endpoint failures', function () {
+    Concurrency::setDefaultInstance('process');
+    metricsServer($this->team, $this->privateKey);
+    metricsServer($this->team, $this->privateKey);
+
+    $component = Livewire::test(ProcessTestableMetrics::class);
+
+    expect($component->get('kpis')['serversOnline'])->toBe(2);
+    expect($component->get('topContainers'))->toHaveCount(2);
+    expect($component->get('chartData')['cpu'])->toBe([[1000, 25.0]]);
+    expect($component->get('chartData')['memory'])->toBe([[1000, 200.0]]);
+    expect($component->get('chartData')['disk'])->toBe([]);
+    expect($component->get('chartData')['networkRx'])->toBe([[1000, 10.0]]);
+});
+
+it('clears stale metrics after the last server is deleted', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+    StubMetricsClient::$containers = [$server->uuid => [['id' => 'deleted-container', 'cpu' => 10]]];
+    $component = Livewire::test(TestableMetrics::class);
+
+    $server->delete();
+    $component->set('containerMetric', 'memory');
+
+    expect($component->get('topContainers'))->toBe([]);
+    expect($component->get('serverRows'))->toBe([]);
+});
+
+it('repairs the Sentinel token before creating worker clients', function () {
+    $server = metricsServer($this->team, $this->privateKey);
+    $server->settings->updateQuietly(['sentinel_token' => 'invalid token']);
+    app()->bind(SentinelMetricsClient::class, fn ($app, $parameters) => new ConcurrentMetricsClient($parameters['server']));
+
+    loadLazy(Livewire::test(Metrics::class));
+
+    $token = $server->settings->fresh()->sentinel_token;
+    expect(ServerSetting::isValidSentinelToken($token))->toBeTrue();
+    expect($token)->not->toBe('invalid token');
+});

tests/Support/ConcurrentMetricsClient.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Tests\Support;
+
+use App\Services\SentinelMetricsClient;
+
+class ConcurrentMetricsClient extends SentinelMetricsClient
+{
+    public function summary(): ?array
+    {
+        return [
+            'online' => true, 'cpu' => 25.0, 'memUsed' => 100, 'memTotal' => 200,
+            'diskUsed' => 10, 'diskTotal' => 100, 'diskPercent' => 10.0,
+            'load1' => 0.4, 'netRx' => 5.0, 'netTx' => 1.0,
+        ];
+    }
+
+    public function containersCurrent(): array
+    {
+        return [['id' => 'container-'.$this->server->uuid, 'cpu' => 25.0]];
+    }
+
+    public function history(string $metric, string $range): array
+    {
+        if ($metric === 'disk') {
+            throw new \RuntimeException('Disk endpoint unavailable.');
+        }
+
+        return [[1000, $metric === 'cpu' ? 25.0 : 100.0]];
+    }
+
+    public function networkHistory(string $range): array
+    {
+        return ['rx' => [[1000, 5.0]], 'tx' => [[1000, 1.0]]];
+    }
+}

tests/Unit/Metrics/HasMetricsMappingTest.php

@@ -59,3 +59,17 @@ it('returns null for a metric when metrics are unavailable', function () {
     expect($d->getNetworkMetrics(5))->toBeNull();
     expect($d->getLoadMetrics(5))->toBeNull();
 });
+
+it('does not use non-root mounts when root disk samples are missing', function () {
+    $double = new MetricsMappingDouble;
+    $double->rowsByType['disk'] = [['time' => 1000, 'mount' => '/boot', 'usedPercent' => 90]];
+
+    expect($double->getDiskMetrics(5))->toBe([]);
+});
+
+it('supports older disk rows without a mount field', function () {
+    $double = new MetricsMappingDouble;
+    $double->rowsByType['disk'] = [['time' => 1000, 'usedPercent' => 40]];
+
+    expect($double->getDiskMetrics(5))->toBe([[1000, 40.0]]);
+});

tests/Unit/Services/SentinelMetricsClientTest.php

@@ -152,3 +152,30 @@ it('rounds the range start to the minute so history responses are cacheable', fu
 
     Carbon::setTestNow();
 });
+
+it('does not substitute another mount when root disk history is absent', function () {
+    $client = fakeMetricsClient();
+    $client->bodies = ['/disk/history' => json_encode([
+        ['time' => 1700000000000, 'mount' => '/boot', 'usedPercent' => 90],
+    ])];
+
+    expect($client->history('disk', '24h'))->toBe([]);
+});
+
+it('supports disk history without mount fields', function () {
+    $client = fakeMetricsClient();
+    $client->bodies = ['/disk/history' => json_encode([
+        ['time' => 1700000000000, 'usedPercent' => 40],
+    ])];
+
+    expect($client->history('disk', '24h'))->toBe([[1699999800000, 40.0]]);
+});
+
+it('does not substitute another disk mount in the summary', function () {
+    $client = fakeMetricsClient();
+    $client->bodies = ['/summary' => json_encode(['disk' => [
+        ['mount' => '/boot', 'used' => 90, 'total' => 100, 'usedPercent' => 90],
+    ]])];
+
+    expect($client->summary()['diskPercent'])->toBeNull();
+});