From e144b990c30e7b810c27fa67cf71db0799d87ecf Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:24:54 +0200 Subject: [PATCH] feat(horizon): add fixed-size per-queue worker pools on Coolify Cloud Cloud production now runs one supervisor per queue (deployments, crons, high, default, maintenance), so a busy queue cannot starve the others. Add maintenance_queue() for slow Docker cleanups and a shared horizon.worker_timeout config value. Self-hosted keeps the s6 supervisor. --- .env.development.example | 9 + app/Actions/Database/StartDatabaseImport.php | 2 +- bootstrap/helpers/shared.php | 25 ++- config/horizon.php | 107 ++++++++---- docs/cloud-horizon-workers.md | 157 +++++++++++++++++ .../DatabaseImportInterruptedCleanupTest.php | 2 +- tests/Feature/HorizonConfigurationTest.php | 162 ++++++++++++++++++ tests/Feature/VolumeBackupTest.php | 4 +- 8 files changed, 425 insertions(+), 43 deletions(-) create mode 100644 docs/cloud-horizon-workers.md create mode 100644 tests/Feature/HorizonConfigurationTest.php diff --git a/.env.development.example b/.env.development.example index f53fcce5f7..4f4198ddb2 100644 --- a/.env.development.example +++ b/.env.development.example @@ -31,6 +31,15 @@ DB_PORT=5432 +# Coolify Cloud Horizon pools (only used when SELF_HOSTED=false and APP_ENV=production). +# Fixed worker count per node and queue. See docs/cloud-horizon-workers.md. +# HORIZON_DEPLOYMENTS_PROCESSES=60 +# HORIZON_CRONS_PROCESSES=60 +# HORIZON_HIGH_PROCESSES=60 +# HORIZON_DEFAULT_PROCESSES=40 +# Docker cleanups; small on purpose to bound concurrent slow remote prunes. +# HORIZON_MAINTENANCE_PROCESSES=10 + # Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local). # Force on in any environment (including production): SERVER_TIMING_ENABLED=true # Force off even in local: SERVER_TIMING_ENABLED=false diff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.php index 5e113d163c..235399e652 100644 --- a/app/Actions/Database/StartDatabaseImport.php +++ b/app/Actions/Database/StartDatabaseImport.php @@ -47,7 +47,7 @@ class StartDatabaseImport public static function jobTimeoutSeconds(): int { $timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS; - $workerTimeout = (int) config('horizon.defaults.s6.timeout'); + $workerTimeout = (int) config('horizon.worker_timeout'); return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout; } diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index cf620bb964..da6faa43ee 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1028,8 +1028,8 @@ function isCloud(): bool * is decided by `isCloud()` (config-based) rather than `HORIZON_QUEUES`, so the dispatching * process needs no special env — only the worker must be configured to drain `deployments`. * - * IMPORTANT: on cloud a worker MUST include `deployments` in its `HORIZON_QUEUES`, otherwise - * these jobs are never processed. + * On cloud, config/horizon.php provisions a dedicated `deployments` pool in production + * (see docs/cloud-horizon-workers.md). */ function deployment_queue(): string { @@ -1045,14 +1045,31 @@ function deployment_queue(): string * by `isCloud()` (config-based), so the dispatching process needs no special env — only the * worker must be configured to drain `crons`. * - * IMPORTANT: on cloud a worker MUST include `crons` in its `HORIZON_QUEUES`, otherwise these - * jobs are never processed. + * On cloud, config/horizon.php provisions a dedicated `crons` pool in production + * (see docs/cloud-horizon-workers.md). */ function crons_queue(): string { return isCloud() ? 'crons' : 'high'; } +/** + * Resolve the queue used for slow server maintenance — scheduled, manual and stop-triggered + * Docker cleanups. + * + * On cloud these jobs run on a dedicated `maintenance` queue so a small, bounded Horizon pool + * drains them and slow remote prunes cannot occupy the `high` workers; self-hosted keeps them + * on the shared `high` queue, so a custom `HORIZON_QUEUES` does not need a new queue name. Routing is decided by `isCloud()` (config-based), so the dispatching + * process needs no special env — only the worker must be configured to drain `maintenance`. + * + * On cloud, config/horizon.php provisions a dedicated `maintenance` pool in production + * (see docs/cloud-horizon-workers.md). + */ +function maintenance_queue(): string +{ + return isCloud() ? 'maintenance' : 'high'; +} + function translate_cron_expression($expression_to_validate): string { if (isset(VALID_CRON_STRINGS[$expression_to_validate])) { diff --git a/config/horizon.php b/config/horizon.php index 0c79f0eed3..cd532f936c 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -3,6 +3,67 @@ use App\Models\ScheduledVolumeBackup; use Illuminate\Support\Str; +/* +| Cloud mode is read from SELF_HOSTED directly instead of isCloud(): isCloud() +| reads the config repository, which is still being built while this file loads. +*/ +$isCloud = ! env('SELF_HOSTED', true); + +$workerOptions = [ + 'connection' => 'redis', + 'maxTime' => env('HORIZON_MAX_TIME', 0), + 'maxJobs' => 400, + 'memory' => 128, + 'tries' => 1, + 'nice' => 0, + 'sleep' => 3, + 'timeout' => min( + max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600), + 85800, + ), +]; + +$selfHostedSupervisor = $workerOptions + [ + 'balance' => env('HORIZON_BALANCE', 'false'), + 'queue' => env('HORIZON_QUEUES', 'high,default'), + 'autoScalingStrategy' => 'size', + 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), + 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4), + 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), + 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), +]; + +/* +| Coolify Cloud: one fixed-size pool per queue, so a busy queue cannot starve +| the others. Process counts apply to each node. Invalid values (not a +| positive integer) fall back to the default. +| +| The `maintenance` pool (see maintenance_queue()) runs slow remote Docker +| cleanups. It is small on purpose: it bounds how many cleanups run at once +| per node, and it comes on top of the other pools, so cleanups never take +| deployment, cron or high workers. +*/ +$cloudSupervisors = []; + +foreach ([ + 'deployments' => ['HORIZON_DEPLOYMENTS_PROCESSES', 60], + 'crons' => ['HORIZON_CRONS_PROCESSES', 60], + 'high' => ['HORIZON_HIGH_PROCESSES', 60], + 'default' => ['HORIZON_DEFAULT_PROCESSES', 40], + 'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10], +] as $queue => [$processesEnv, $defaultProcesses]) { + $processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, [ + 'options' => ['min_range' => 1, 'default' => $defaultProcesses], + ]); + + $cloudSupervisors[$queue] = $workerOptions + [ + 'queue' => $queue, + 'balance' => false, + 'minProcesses' => $processes, + 'maxProcesses' => $processes, + ]; +} + return [ /* @@ -190,45 +251,21 @@ return [ | in all environments. These supervisors and settings handle all your | queued jobs and will be provisioned by Horizon during deployment. | + | "defaults" stays empty: Horizon merges every default supervisor into + | every environment, so each environment defines complete options. + | */ - 'defaults' => [ - 's6' => [ - 'connection' => 'redis', - 'balance' => env('HORIZON_BALANCE', 'false'), - 'queue' => env('HORIZON_QUEUES', 'high,default'), - 'maxTime' => env('HORIZON_MAX_TIME', 0), - 'maxJobs' => 400, - 'memory' => 128, - 'tries' => 1, - 'nice' => 0, - 'sleep' => 3, - 'timeout' => min( - max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600), - 85800, - ), - ], + 'defaults' => [], - ], + /* + | Worker timeout shared by every supervisor. Jobs that must finish before + | the worker kills them (for example database imports) read this value. + */ + 'worker_timeout' => $workerOptions['timeout'], 'environments' => [ - 'production' => [ - 's6' => [ - 'autoScalingStrategy' => 'size', - 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), - 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4), - 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), - 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), - ], - ], - 'local' => [ - 's6' => [ - 'autoScalingStrategy' => 'size', - 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), - 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4), - 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), - 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), - ], - ], + 'production' => $isCloud ? $cloudSupervisors : ['s6' => $selfHostedSupervisor], + 'local' => ['s6' => $selfHostedSupervisor], ], ]; diff --git a/docs/cloud-horizon-workers.md b/docs/cloud-horizon-workers.md new file mode 100644 index 0000000000..11ee81ef5e --- /dev/null +++ b/docs/cloud-horizon-workers.md @@ -0,0 +1,157 @@ +# Coolify Cloud Horizon worker pools + +Coolify Cloud runs Horizon on two app nodes that share one Redis. On Cloud +(`SELF_HOSTED=false`, `APP_ENV=production`), `config/horizon.php` provisions +one fixed-size supervisor per queue. A busy queue cannot starve the others. + +| Supervisor | Queue | Env variable | Default per node | +|---------------|---------------|---------------------------------|------------------| +| `deployments` | `deployments` | `HORIZON_DEPLOYMENTS_PROCESSES` | 60 | +| `crons` | `crons` | `HORIZON_CRONS_PROCESSES` | 60 | +| `high` | `high` | `HORIZON_HIGH_PROCESSES` | 60 | +| `default` | `default` | `HORIZON_DEFAULT_PROCESSES` | 40 | +| `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10 | + +- Counts apply to **each node**. Two nodes with the defaults run 460 workers. +- `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup + that runs after a resource stops). Remote prunes can be slow, so this pool + is small on purpose: it limits how many cleanups run at the same time and + keeps them away from the `high` workers. It is added on top of the other + pools. Each cleanup has a 600 s timeout, and only one cleanup runs per + server at a time. Self-hosted instances keep cleanups on `high`. +- Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon + does not scale them. +- A value that is not a positive integer falls back to the default. +- `HORIZON_QUEUES`, `HORIZON_MIN_PROCESSES`, `HORIZON_MAX_PROCESSES`, and the + `HORIZON_BALANCE*` variables have no effect on Cloud production. They still + configure the self-hosted `s6` supervisor and the `local` environment. +- Timeout, retry, memory, and recycling settings (`HORIZON_TIMEOUT`, + `HORIZON_MAX_TIME`, `maxJobs`, `memory`, `tries`) are the same as for + self-hosted. +- The config reads `SELF_HOSTED` with `env()` and does not call `isCloud()`, + because `isCloud()` reads the config repository, which is not complete while + config files load. + +## Sample Cloud configuration + +Put these values in the `.env` of each node. The values are the defaults, so +set only the values that you change. + +```dotenv +SELF_HOSTED=false +APP_ENV=production +HORIZON_DEPLOYMENTS_PROCESSES=60 +HORIZON_CRONS_PROCESSES=60 +HORIZON_HIGH_PROCESSES=60 +HORIZON_DEFAULT_PROCESSES=40 +HORIZON_MAINTENANCE_PROCESSES=10 +# Worker timeout. Must stay above the longest job timeout (36000 s) and below +# retry_after (86400 s). The config clamps it to 36600..85800. +HORIZON_TIMEOUT=36600 +``` + +Redis that holds queues must use `maxmemory-policy noeviction`. An evicting +policy deletes queued jobs without an error. + +## Worker shutdown, retry_after, and interrupted jobs + +- **Graceful (`php artisan horizon:terminate`)**: workers stop taking new + jobs and finish the current job. `fast_termination` is `false`, so the + master waits until the last worker exits. This can take up to the longest + job timeout. Waiting jobs stay in Redis. The other node continues to + process them. +- **Forced (SIGKILL, container stop after its grace period, OOM kill)**: the + running job stays in `queues::reserved`. Redis gives it to a worker + again only after `retry_after` (86400 s). Then the job has attempt 2: + - Jobs with `tries = 1` fail with `MaxAttemptsExceededException` and their + `failed()` method runs. + - A `ScheduledTaskJob` does not run its command again. It fails. + - A scheduled occurrence that a killed worker claimed is marked `failed` + ("interrupted") when it is older than the worker timeout plus 15 minutes. + It is not run again, because part of its remote work can be done. +- Do not lower `retry_after` below the worker timeout. A running job would + then be given to a second worker. + +### Which settings need new workers + +| Change | Needed action | +|-----------------------------------------------------------|--------------------------------------------------| +| `HORIZON_*_PROCESSES`, `HORIZON_TIMEOUT`, queue names | `config:cache` and `horizon:terminate` | +| Horizon `trim` values | `config:cache` and `horizon:terminate` | +| `retry_after` (`config/queue.php`) | `config:cache` and `horizon:terminate` | +| Application code (jobs, queues, middleware) | New image, then `horizon:terminate` | +| New database migration | `php artisan migrate` before new workers start | + +## Rollout + +Do these steps on one node at a time. Do not flush Redis or clear queues. + +1. Run the database migrations once (the new volume backup recovery columns + must exist before the new scheduler runs): + + ```bash + docker exec coolify php artisan migrate --force + ``` + +2. Optional: set the `HORIZON_*_PROCESSES` variables in the node's `.env`. + Remove manual pool overrides from earlier hotfixes. +3. Rebuild the configuration cache: + + ```bash + docker exec coolify php artisan config:cache + ``` + +4. Restart Horizon on this node only. `horizon:terminate` lets the running + jobs finish, then s6 starts Horizon again with the new configuration: + + ```bash + docker exec coolify php artisan horizon:terminate + ``` + +5. Make sure the five supervisors are running on this node: + + ```bash + docker exec coolify php artisan horizon:supervisors + ``` + +6. Make sure the `maintenance` queue drains (its depth goes down) and that + `php artisan scheduled:diagnostics` shows no growing count of stale + occurrences. +7. Do the same steps on the next node. + +Jobs that the old code put on `high` (for example Docker cleanups) stay on +`high`, and the `high` pool still processes them. + +## Rollback + +1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides. +2. Run `config:cache` and `horizon:terminate` on one node at a time. +3. Jobs that wait on the `maintenance` queue are not lost, but the old + configuration has no `maintenance` pool. Keep one node with the new + configuration until `maintenance` is empty, or start a temporary worker: + `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`. +4. Do not roll back the migration. The old code ignores the new columns. + +## Metrics to collect before you change worker counts + +The incident samples are short and show only completed jobs. Collect these +values before you change pool sizes: + +- Arrival and completion rate per queue (per minute). +- Age of the oldest waiting job per queue (`LINDEX queues: -1`, then + read `pushedAt` from the payload). Do not scan the full list. +- Queue depth (`LLEN queues:`) and reserved age + (`ZRANGE queues::reserved 0 0 WITHSCORES`). +- Active worker processes per supervisor (count processes, not the + Horizon reserved count; reserved entries stay after forced restarts). +- Job duration distribution (p50/p95/max) and failure/retry rate per job + class. +- Stale scheduled occurrences per status (`scheduled:diagnostics` and the + `open_occurrences_over_15_minutes` value in `scheduled.log`). +- Redis `used_memory` / `maxmemory`, the delta of `evicted_keys` (`INFO + stats`), and the delta of `errorstat_OOM` (`INFO errorstats`). +- Horizon metadata size: the count of `horizon:*` job hashes and the + `horizon:recent_jobs` / `horizon:completed_jobs` sizes. + +Label metrics only by queue and job class. Do not label by resource ID or +occurrence UUID. diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.php index 0ef66fed33..f82b8cc438 100644 --- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php +++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php @@ -289,7 +289,7 @@ test('an import task may run longer than the SSH command timeout while other rem $timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [ (data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout, ]); - $workerTimeout = (int) config('horizon.defaults.s6.timeout'); + $workerTimeout = (int) config('horizon.worker_timeout'); expect($timeouts['other'])->toBe(600) ->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600)) diff --git a/tests/Feature/HorizonConfigurationTest.php b/tests/Feature/HorizonConfigurationTest.php new file mode 100644 index 0000000000..4f8a8d6fc3 --- /dev/null +++ b/tests/Feature/HorizonConfigurationTest.php @@ -0,0 +1,162 @@ + $variables + */ +function loadHorizonConfigWithEnv(array $variables): array +{ + $previous = []; + + foreach ($variables as $name => $value) { + $previous[$name] = getenv($name); + $value === null ? putenv($name) : putenv("{$name}={$value}"); + if ($value === null) { + unset($_ENV[$name], $_SERVER[$name]); + } else { + $_ENV[$name] = $_SERVER[$name] = $value; + } + } + + try { + return require config_path('horizon.php'); + } finally { + foreach ($previous as $name => $value) { + $value === false ? putenv($name) : putenv("{$name}={$value}"); + if ($value === false) { + unset($_ENV[$name], $_SERVER[$name]); + } else { + $_ENV[$name] = $_SERVER[$name] = $value; + } + } + } +} + +/** + * Parse every environment through Horizon's ProvisioningPlan, as `php artisan horizon` does on startup. + * + * @return array> + */ +function horizonSupervisorsFor(array $config): array +{ + $parsed = array_map( + fn ($supervisors) => collect($supervisors)->all(), + (new ProvisioningPlan('master', $config['environments'], $config['defaults']))->toSupervisorOptions(), + ); + + foreach ($parsed as $supervisors) { + foreach ($supervisors as $options) { + expect($options->connection)->toBe('redis') + ->and($options->timeout)->toBe($config['worker_timeout']) + ->and($options->toSupervisorCommand())->toContain('horizon:supervisor'); + } + } + + return $parsed; +} + +test('self-hosted keeps a single auto-scaling s6 supervisor in every environment', function () { + $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([ + 'SELF_HOSTED' => 'true', + 'HORIZON_QUEUES' => null, + 'HORIZON_MIN_PROCESSES' => null, + 'HORIZON_MAX_PROCESSES' => null, + ])); + + expect(array_keys($parsed))->toBe(['production', 'local']); + + foreach ($parsed as $supervisors) { + expect(array_keys($supervisors))->toBe(['s6']); + + $s6 = $supervisors['s6']; + expect($s6->queue)->toBe('high,default') + ->and($s6->minProcesses)->toBe(1) + ->and($s6->maxProcesses)->toBe(4) + ->and($s6->autoScalingStrategy)->toBe('size') + ->and($s6->maxJobs)->toBe(400) + ->and($s6->memory)->toBe(128) + ->and($s6->maxTries)->toBe(1); + } +}); + +test('cloud production runs one fixed-size pool per queue with default process counts', function () { + $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([ + 'SELF_HOSTED' => 'false', + 'HORIZON_DEPLOYMENTS_PROCESSES' => null, + 'HORIZON_CRONS_PROCESSES' => null, + 'HORIZON_HIGH_PROCESSES' => null, + 'HORIZON_DEFAULT_PROCESSES' => null, + 'HORIZON_MAINTENANCE_PROCESSES' => null, + ])); + + $production = $parsed['production']; + expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']); + + foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) { + $pool = $production[$queue]; + expect($pool->queue)->toBe($queue) + ->and($pool->balancing())->toBeFalse() + ->and($pool->minProcesses)->toBe($processes) + ->and($pool->maxProcesses)->toBe($processes) + ->and($pool->maxJobs)->toBe(400) + ->and($pool->memory)->toBe(128) + ->and($pool->maxTries)->toBe(1) + ->and($pool->timeout)->toBe($parsed['local']['s6']->timeout); + } + + expect(array_keys($parsed['local']))->toBe(['s6']); +}); + +test('cloud pool sizes come from env and invalid values fall back to defaults', function () { + $production = horizonSupervisorsFor(loadHorizonConfigWithEnv([ + 'SELF_HOSTED' => 'false', + 'HORIZON_DEPLOYMENTS_PROCESSES' => '25', + 'HORIZON_CRONS_PROCESSES' => '0', + 'HORIZON_HIGH_PROCESSES' => 'abc', + 'HORIZON_DEFAULT_PROCESSES' => '-5', + 'HORIZON_MAINTENANCE_PROCESSES' => '4', + ]))['production']; + + expect($production['deployments']->maxProcesses)->toBe(25) + ->and($production['deployments']->minProcesses)->toBe(25) + ->and($production['crons']->maxProcesses)->toBe(60) + ->and($production['high']->maxProcesses)->toBe(60) + ->and($production['default']->maxProcesses)->toBe(40) + ->and($production['maintenance']->minProcesses)->toBe(4) + ->and($production['maintenance']->maxProcesses)->toBe(4); + + $fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv([ + 'SELF_HOSTED' => 'false', + 'HORIZON_MAINTENANCE_PROCESSES' => 'ten', + ]))['production']; + + expect($fallback['maintenance']->maxProcesses)->toBe(10); +}); + +test('cloud queue helpers route to queues that have a dedicated pool', function () { + config(['constants.coolify.self_hosted' => false]); + + $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production']; + + expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]); +}); + +test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () { + config(['constants.coolify.self_hosted' => true]); + + $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([ + 'SELF_HOSTED' => 'true', + 'HORIZON_QUEUES' => null, + 'HORIZON_MAINTENANCE_PROCESSES' => '3', + ])); + + foreach ($parsed as $supervisors) { + expect(array_keys($supervisors))->toBe(['s6']) + ->and(explode(',', $supervisors['s6']->queue)) + ->toContain(deployment_queue(), crons_queue(), maintenance_queue()); + } +}); diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php index 1f733f72d4..2fc39486d5 100644 --- a/tests/Feature/VolumeBackupTest.php +++ b/tests/Feature/VolumeBackupTest.php @@ -61,8 +61,8 @@ it('allows large volume backups to run for ten hours by default', function () { expect($job->timeout)->toBe(36000) ->and((new VolumeBackups)->timeout)->toBe(36000) - ->and(config('horizon.defaults.s6.timeout'))->toBeGreaterThan($job->timeout) - ->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.defaults.s6.timeout')); + ->and(config('horizon.worker_timeout'))->toBeGreaterThan($job->timeout) + ->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.worker_timeout')); }); it('changes the default volume backup timeout without changing existing timeouts', function () {