feat(horizon): add fixed-size per-queue worker pools on Coolify Cloud

Cloud production now runs one supervisor per queue (deployments, crons,
high, default, maintenance), so a busy queue cannot starve the others.
Add maintenance_queue() for slow Docker cleanups and a shared
horizon.worker_timeout config value. Self-hosted keeps the s6 supervisor.

.env.development.example

@@ -31,6 +31,15 @@ DB_PORT=5432
 
 
 
+# Coolify Cloud Horizon pools (only used when SELF_HOSTED=false and APP_ENV=production).
+# Fixed worker count per node and queue. See docs/cloud-horizon-workers.md.
+# HORIZON_DEPLOYMENTS_PROCESSES=60
+# HORIZON_CRONS_PROCESSES=60
+# HORIZON_HIGH_PROCESSES=60
+# HORIZON_DEFAULT_PROCESSES=40
+# Docker cleanups; small on purpose to bound concurrent slow remote prunes.
+# HORIZON_MAINTENANCE_PROCESSES=10
+
 # Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local).
 # Force on in any environment (including production): SERVER_TIMING_ENABLED=true
 # Force off even in local: SERVER_TIMING_ENABLED=false

app/Actions/Database/StartDatabaseImport.php

@@ -47,7 +47,7 @@ class StartDatabaseImport
     public static function jobTimeoutSeconds(): int
     {
         $timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS;
-        $workerTimeout = (int) config('horizon.defaults.s6.timeout');
+        $workerTimeout = (int) config('horizon.worker_timeout');
 
         return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout;
     }

bootstrap/helpers/shared.php

@@ -1028,8 +1028,8 @@ function isCloud(): bool
  * is decided by `isCloud()` (config-based) rather than `HORIZON_QUEUES`, so the dispatching
  * process needs no special env — only the worker must be configured to drain `deployments`.
  *
- * IMPORTANT: on cloud a worker MUST include `deployments` in its `HORIZON_QUEUES`, otherwise
- * these jobs are never processed.
+ * On cloud, config/horizon.php provisions a dedicated `deployments` pool in production
+ * (see docs/cloud-horizon-workers.md).
  */
 function deployment_queue(): string
 {
@@ -1045,14 +1045,31 @@ function deployment_queue(): string
  * by `isCloud()` (config-based), so the dispatching process needs no special env — only the
  * worker must be configured to drain `crons`.
  *
- * IMPORTANT: on cloud a worker MUST include `crons` in its `HORIZON_QUEUES`, otherwise these
- * jobs are never processed.
+ * On cloud, config/horizon.php provisions a dedicated `crons` pool in production
+ * (see docs/cloud-horizon-workers.md).
  */
 function crons_queue(): string
 {
     return isCloud() ? 'crons' : 'high';
 }
 
+/**
+ * Resolve the queue used for slow server maintenance — scheduled, manual and stop-triggered
+ * Docker cleanups.
+ *
+ * On cloud these jobs run on a dedicated `maintenance` queue so a small, bounded Horizon pool
+ * drains them and slow remote prunes cannot occupy the `high` workers; self-hosted keeps them
+ * on the shared `high` queue, so a custom `HORIZON_QUEUES` does not need a new queue name. Routing is decided by `isCloud()` (config-based), so the dispatching
+ * process needs no special env — only the worker must be configured to drain `maintenance`.
+ *
+ * On cloud, config/horizon.php provisions a dedicated `maintenance` pool in production
+ * (see docs/cloud-horizon-workers.md).
+ */
+function maintenance_queue(): string
+{
+    return isCloud() ? 'maintenance' : 'high';
+}
+
 function translate_cron_expression($expression_to_validate): string
 {
     if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {

config/horizon.php

@@ -3,6 +3,67 @@
 use App\Models\ScheduledVolumeBackup;
 use Illuminate\Support\Str;
 
+/*
+| Cloud mode is read from SELF_HOSTED directly instead of isCloud(): isCloud()
+| reads the config repository, which is still being built while this file loads.
+*/
+$isCloud = ! env('SELF_HOSTED', true);
+
+$workerOptions = [
+    'connection' => 'redis',
+    'maxTime' => env('HORIZON_MAX_TIME', 0),
+    'maxJobs' => 400,
+    'memory' => 128,
+    'tries' => 1,
+    'nice' => 0,
+    'sleep' => 3,
+    'timeout' => min(
+        max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600),
+        85800,
+    ),
+];
+
+$selfHostedSupervisor = $workerOptions + [
+    'balance' => env('HORIZON_BALANCE', 'false'),
+    'queue' => env('HORIZON_QUEUES', 'high,default'),
+    'autoScalingStrategy' => 'size',
+    'minProcesses' => env('HORIZON_MIN_PROCESSES', 1),
+    'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4),
+    'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
+    'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
+];
+
+/*
+| Coolify Cloud: one fixed-size pool per queue, so a busy queue cannot starve
+| the others. Process counts apply to each node. Invalid values (not a
+| positive integer) fall back to the default.
+|
+| The `maintenance` pool (see maintenance_queue()) runs slow remote Docker
+| cleanups. It is small on purpose: it bounds how many cleanups run at once
+| per node, and it comes on top of the other pools, so cleanups never take
+| deployment, cron or high workers.
+*/
+$cloudSupervisors = [];
+
+foreach ([
+    'deployments' => ['HORIZON_DEPLOYMENTS_PROCESSES', 60],
+    'crons' => ['HORIZON_CRONS_PROCESSES', 60],
+    'high' => ['HORIZON_HIGH_PROCESSES', 60],
+    'default' => ['HORIZON_DEFAULT_PROCESSES', 40],
+    'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10],
+] as $queue => [$processesEnv, $defaultProcesses]) {
+    $processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, [
+        'options' => ['min_range' => 1, 'default' => $defaultProcesses],
+    ]);
+
+    $cloudSupervisors[$queue] = $workerOptions + [
+        'queue' => $queue,
+        'balance' => false,
+        'minProcesses' => $processes,
+        'maxProcesses' => $processes,
+    ];
+}
+
 return [
 
     /*
@@ -190,45 +251,21 @@ return [
     | in all environments. These supervisors and settings handle all your
     | queued jobs and will be provisioned by Horizon during deployment.
     |
+    | "defaults" stays empty: Horizon merges every default supervisor into
+    | every environment, so each environment defines complete options.
+    |
     */
 
-    'defaults' => [
-        's6' => [
-            'connection' => 'redis',
-            'balance' => env('HORIZON_BALANCE', 'false'),
-            'queue' => env('HORIZON_QUEUES', 'high,default'),
-            'maxTime' => env('HORIZON_MAX_TIME', 0),
-            'maxJobs' => 400,
-            'memory' => 128,
-            'tries' => 1,
-            'nice' => 0,
-            'sleep' => 3,
-            'timeout' => min(
-                max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600),
-                85800,
-            ),
-        ],
+    'defaults' => [],
 
-    ],
+    /*
+    | Worker timeout shared by every supervisor. Jobs that must finish before
+    | the worker kills them (for example database imports) read this value.
+    */
+    'worker_timeout' => $workerOptions['timeout'],
 
     'environments' => [
-        'production' => [
-            's6' => [
-                'autoScalingStrategy' => 'size',
-                'minProcesses' => env('HORIZON_MIN_PROCESSES', 1),
-                'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4),
-                'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
-                'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
-            ],
-        ],
-        'local' => [
-            's6' => [
-                'autoScalingStrategy' => 'size',
-                'minProcesses' => env('HORIZON_MIN_PROCESSES', 1),
-                'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4),
-                'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
-                'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
-            ],
-        ],
+        'production' => $isCloud ? $cloudSupervisors : ['s6' => $selfHostedSupervisor],
+        'local' => ['s6' => $selfHostedSupervisor],
     ],
 ];

docs/cloud-horizon-workers.md

@@ -0,0 +1,157 @@
+# Coolify Cloud Horizon worker pools
+
+Coolify Cloud runs Horizon on two app nodes that share one Redis. On Cloud
+(`SELF_HOSTED=false`, `APP_ENV=production`), `config/horizon.php` provisions
+one fixed-size supervisor per queue. A busy queue cannot starve the others.
+
+| Supervisor    | Queue         | Env variable                    | Default per node |
+|---------------|---------------|---------------------------------|------------------|
+| `deployments` | `deployments` | `HORIZON_DEPLOYMENTS_PROCESSES` | 60               |
+| `crons`       | `crons`       | `HORIZON_CRONS_PROCESSES`       | 60               |
+| `high`        | `high`        | `HORIZON_HIGH_PROCESSES`        | 60               |
+| `default`     | `default`     | `HORIZON_DEFAULT_PROCESSES`     | 40               |
+| `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10               |
+
+- Counts apply to **each node**. Two nodes with the defaults run 460 workers.
+- `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup
+  that runs after a resource stops). Remote prunes can be slow, so this pool
+  is small on purpose: it limits how many cleanups run at the same time and
+  keeps them away from the `high` workers. It is added on top of the other
+  pools. Each cleanup has a 600 s timeout, and only one cleanup runs per
+  server at a time. Self-hosted instances keep cleanups on `high`.
+- Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon
+  does not scale them.
+- A value that is not a positive integer falls back to the default.
+- `HORIZON_QUEUES`, `HORIZON_MIN_PROCESSES`, `HORIZON_MAX_PROCESSES`, and the
+  `HORIZON_BALANCE*` variables have no effect on Cloud production. They still
+  configure the self-hosted `s6` supervisor and the `local` environment.
+- Timeout, retry, memory, and recycling settings (`HORIZON_TIMEOUT`,
+  `HORIZON_MAX_TIME`, `maxJobs`, `memory`, `tries`) are the same as for
+  self-hosted.
+- The config reads `SELF_HOSTED` with `env()` and does not call `isCloud()`,
+  because `isCloud()` reads the config repository, which is not complete while
+  config files load.
+
+## Sample Cloud configuration
+
+Put these values in the `.env` of each node. The values are the defaults, so
+set only the values that you change.
+
+```dotenv
+SELF_HOSTED=false
+APP_ENV=production
+HORIZON_DEPLOYMENTS_PROCESSES=60
+HORIZON_CRONS_PROCESSES=60
+HORIZON_HIGH_PROCESSES=60
+HORIZON_DEFAULT_PROCESSES=40
+HORIZON_MAINTENANCE_PROCESSES=10
+# Worker timeout. Must stay above the longest job timeout (36000 s) and below
+# retry_after (86400 s). The config clamps it to 36600..85800.
+HORIZON_TIMEOUT=36600
+```
+
+Redis that holds queues must use `maxmemory-policy noeviction`. An evicting
+policy deletes queued jobs without an error.
+
+## Worker shutdown, retry_after, and interrupted jobs
+
+- **Graceful (`php artisan horizon:terminate`)**: workers stop taking new
+  jobs and finish the current job. `fast_termination` is `false`, so the
+  master waits until the last worker exits. This can take up to the longest
+  job timeout. Waiting jobs stay in Redis. The other node continues to
+  process them.
+- **Forced (SIGKILL, container stop after its grace period, OOM kill)**: the
+  running job stays in `queues:<queue>:reserved`. Redis gives it to a worker
+  again only after `retry_after` (86400 s). Then the job has attempt 2:
+  - Jobs with `tries = 1` fail with `MaxAttemptsExceededException` and their
+    `failed()` method runs.
+  - A `ScheduledTaskJob` does not run its command again. It fails.
+  - A scheduled occurrence that a killed worker claimed is marked `failed`
+    ("interrupted") when it is older than the worker timeout plus 15 minutes.
+    It is not run again, because part of its remote work can be done.
+- Do not lower `retry_after` below the worker timeout. A running job would
+  then be given to a second worker.
+
+### Which settings need new workers
+
+| Change                                                    | Needed action                                    |
+|-----------------------------------------------------------|--------------------------------------------------|
+| `HORIZON_*_PROCESSES`, `HORIZON_TIMEOUT`, queue names      | `config:cache` and `horizon:terminate`           |
+| Horizon `trim` values                                     | `config:cache` and `horizon:terminate`           |
+| `retry_after` (`config/queue.php`)                        | `config:cache` and `horizon:terminate`           |
+| Application code (jobs, queues, middleware)               | New image, then `horizon:terminate`              |
+| New database migration                                    | `php artisan migrate` before new workers start   |
+
+## Rollout
+
+Do these steps on one node at a time. Do not flush Redis or clear queues.
+
+1. Run the database migrations once (the new volume backup recovery columns
+   must exist before the new scheduler runs):
+
+   ```bash
+   docker exec coolify php artisan migrate --force
+   ```
+
+2. Optional: set the `HORIZON_*_PROCESSES` variables in the node's `.env`.
+   Remove manual pool overrides from earlier hotfixes.
+3. Rebuild the configuration cache:
+
+   ```bash
+   docker exec coolify php artisan config:cache
+   ```
+
+4. Restart Horizon on this node only. `horizon:terminate` lets the running
+   jobs finish, then s6 starts Horizon again with the new configuration:
+
+   ```bash
+   docker exec coolify php artisan horizon:terminate
+   ```
+
+5. Make sure the five supervisors are running on this node:
+
+   ```bash
+   docker exec coolify php artisan horizon:supervisors
+   ```
+
+6. Make sure the `maintenance` queue drains (its depth goes down) and that
+   `php artisan scheduled:diagnostics` shows no growing count of stale
+   occurrences.
+7. Do the same steps on the next node.
+
+Jobs that the old code put on `high` (for example Docker cleanups) stay on
+`high`, and the `high` pool still processes them.
+
+## Rollback
+
+1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides.
+2. Run `config:cache` and `horizon:terminate` on one node at a time.
+3. Jobs that wait on the `maintenance` queue are not lost, but the old
+   configuration has no `maintenance` pool. Keep one node with the new
+   configuration until `maintenance` is empty, or start a temporary worker:
+   `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`.
+4. Do not roll back the migration. The old code ignores the new columns.
+
+## Metrics to collect before you change worker counts
+
+The incident samples are short and show only completed jobs. Collect these
+values before you change pool sizes:
+
+- Arrival and completion rate per queue (per minute).
+- Age of the oldest waiting job per queue (`LINDEX queues:<queue> -1`, then
+  read `pushedAt` from the payload). Do not scan the full list.
+- Queue depth (`LLEN queues:<queue>`) and reserved age
+  (`ZRANGE queues:<queue>:reserved 0 0 WITHSCORES`).
+- Active worker processes per supervisor (count processes, not the
+  Horizon reserved count; reserved entries stay after forced restarts).
+- Job duration distribution (p50/p95/max) and failure/retry rate per job
+  class.
+- Stale scheduled occurrences per status (`scheduled:diagnostics` and the
+  `open_occurrences_over_15_minutes` value in `scheduled.log`).
+- Redis `used_memory` / `maxmemory`, the delta of `evicted_keys` (`INFO
+  stats`), and the delta of `errorstat_OOM` (`INFO errorstats`).
+- Horizon metadata size: the count of `horizon:*` job hashes and the
+  `horizon:recent_jobs` / `horizon:completed_jobs` sizes.
+
+Label metrics only by queue and job class. Do not label by resource ID or
+occurrence UUID.

tests/Feature/DatabaseImportInterruptedCleanupTest.php

@@ -289,7 +289,7 @@ test('an import task may run longer than the SSH command timeout while other rem
     $timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [
         (data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout,
     ]);
-    $workerTimeout = (int) config('horizon.defaults.s6.timeout');
+    $workerTimeout = (int) config('horizon.worker_timeout');
 
     expect($timeouts['other'])->toBe(600)
         ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600))

tests/Feature/HorizonConfigurationTest.php

@@ -0,0 +1,162 @@
+<?php
+
+use Laravel\Horizon\ProvisioningPlan;
+use Laravel\Horizon\SupervisorOptions;
+
+/**
+ * Evaluate config/horizon.php with the given environment variables, then restore them.
+ *
+ * @param  array<string, string|null>  $variables
+ */
+function loadHorizonConfigWithEnv(array $variables): array
+{
+    $previous = [];
+
+    foreach ($variables as $name => $value) {
+        $previous[$name] = getenv($name);
+        $value === null ? putenv($name) : putenv("{$name}={$value}");
+        if ($value === null) {
+            unset($_ENV[$name], $_SERVER[$name]);
+        } else {
+            $_ENV[$name] = $_SERVER[$name] = $value;
+        }
+    }
+
+    try {
+        return require config_path('horizon.php');
+    } finally {
+        foreach ($previous as $name => $value) {
+            $value === false ? putenv($name) : putenv("{$name}={$value}");
+            if ($value === false) {
+                unset($_ENV[$name], $_SERVER[$name]);
+            } else {
+                $_ENV[$name] = $_SERVER[$name] = $value;
+            }
+        }
+    }
+}
+
+/**
+ * Parse every environment through Horizon's ProvisioningPlan, as `php artisan horizon` does on startup.
+ *
+ * @return array<string, array<string, SupervisorOptions>>
+ */
+function horizonSupervisorsFor(array $config): array
+{
+    $parsed = array_map(
+        fn ($supervisors) => collect($supervisors)->all(),
+        (new ProvisioningPlan('master', $config['environments'], $config['defaults']))->toSupervisorOptions(),
+    );
+
+    foreach ($parsed as $supervisors) {
+        foreach ($supervisors as $options) {
+            expect($options->connection)->toBe('redis')
+                ->and($options->timeout)->toBe($config['worker_timeout'])
+                ->and($options->toSupervisorCommand())->toContain('horizon:supervisor');
+        }
+    }
+
+    return $parsed;
+}
+
+test('self-hosted keeps a single auto-scaling s6 supervisor in every environment', function () {
+    $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([
+        'SELF_HOSTED' => 'true',
+        'HORIZON_QUEUES' => null,
+        'HORIZON_MIN_PROCESSES' => null,
+        'HORIZON_MAX_PROCESSES' => null,
+    ]));
+
+    expect(array_keys($parsed))->toBe(['production', 'local']);
+
+    foreach ($parsed as $supervisors) {
+        expect(array_keys($supervisors))->toBe(['s6']);
+
+        $s6 = $supervisors['s6'];
+        expect($s6->queue)->toBe('high,default')
+            ->and($s6->minProcesses)->toBe(1)
+            ->and($s6->maxProcesses)->toBe(4)
+            ->and($s6->autoScalingStrategy)->toBe('size')
+            ->and($s6->maxJobs)->toBe(400)
+            ->and($s6->memory)->toBe(128)
+            ->and($s6->maxTries)->toBe(1);
+    }
+});
+
+test('cloud production runs one fixed-size pool per queue with default process counts', function () {
+    $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([
+        'SELF_HOSTED' => 'false',
+        'HORIZON_DEPLOYMENTS_PROCESSES' => null,
+        'HORIZON_CRONS_PROCESSES' => null,
+        'HORIZON_HIGH_PROCESSES' => null,
+        'HORIZON_DEFAULT_PROCESSES' => null,
+        'HORIZON_MAINTENANCE_PROCESSES' => null,
+    ]));
+
+    $production = $parsed['production'];
+    expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']);
+
+    foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) {
+        $pool = $production[$queue];
+        expect($pool->queue)->toBe($queue)
+            ->and($pool->balancing())->toBeFalse()
+            ->and($pool->minProcesses)->toBe($processes)
+            ->and($pool->maxProcesses)->toBe($processes)
+            ->and($pool->maxJobs)->toBe(400)
+            ->and($pool->memory)->toBe(128)
+            ->and($pool->maxTries)->toBe(1)
+            ->and($pool->timeout)->toBe($parsed['local']['s6']->timeout);
+    }
+
+    expect(array_keys($parsed['local']))->toBe(['s6']);
+});
+
+test('cloud pool sizes come from env and invalid values fall back to defaults', function () {
+    $production = horizonSupervisorsFor(loadHorizonConfigWithEnv([
+        'SELF_HOSTED' => 'false',
+        'HORIZON_DEPLOYMENTS_PROCESSES' => '25',
+        'HORIZON_CRONS_PROCESSES' => '0',
+        'HORIZON_HIGH_PROCESSES' => 'abc',
+        'HORIZON_DEFAULT_PROCESSES' => '-5',
+        'HORIZON_MAINTENANCE_PROCESSES' => '4',
+    ]))['production'];
+
+    expect($production['deployments']->maxProcesses)->toBe(25)
+        ->and($production['deployments']->minProcesses)->toBe(25)
+        ->and($production['crons']->maxProcesses)->toBe(60)
+        ->and($production['high']->maxProcesses)->toBe(60)
+        ->and($production['default']->maxProcesses)->toBe(40)
+        ->and($production['maintenance']->minProcesses)->toBe(4)
+        ->and($production['maintenance']->maxProcesses)->toBe(4);
+
+    $fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv([
+        'SELF_HOSTED' => 'false',
+        'HORIZON_MAINTENANCE_PROCESSES' => 'ten',
+    ]))['production'];
+
+    expect($fallback['maintenance']->maxProcesses)->toBe(10);
+});
+
+test('cloud queue helpers route to queues that have a dedicated pool', function () {
+    config(['constants.coolify.self_hosted' => false]);
+
+    $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production'];
+
+    expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]);
+});
+
+test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () {
+    config(['constants.coolify.self_hosted' => true]);
+
+    $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([
+        'SELF_HOSTED' => 'true',
+        'HORIZON_QUEUES' => null,
+        'HORIZON_MAINTENANCE_PROCESSES' => '3',
+    ]));
+
+    foreach ($parsed as $supervisors) {
+        expect(array_keys($supervisors))->toBe(['s6'])
+            ->and(explode(',', $supervisors['s6']->queue))
+            ->toContain(deployment_queue(), crons_queue(), maintenance_queue());
+    }
+});

tests/Feature/VolumeBackupTest.php

@@ -61,8 +61,8 @@ it('allows large volume backups to run for ten hours by default', function () {
 
     expect($job->timeout)->toBe(36000)
         ->and((new VolumeBackups)->timeout)->toBe(36000)
-        ->and(config('horizon.defaults.s6.timeout'))->toBeGreaterThan($job->timeout)
-        ->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.defaults.s6.timeout'));
+        ->and(config('horizon.worker_timeout'))->toBeGreaterThan($job->timeout)
+        ->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.worker_timeout'));
 });
 
 it('changes the default volume backup timeout without changing existing timeouts', function () {