fix(multi-server): make applications on several servers consistent

- Stop the application on every server, also when one server is not functional.
- Write file mounts on every server of the application, not only on the main server.
- Deploy the exact commit, rollback flag and image tag of the main deployment to the
  additional servers, so a rollback reverts all servers.
- Pull the pushed image on additional servers for inline Dockerfile applications
  instead of building it again on each server.
- Fail multi-server deployments without a Docker image name, also from webhooks and the API.
- Send one deployment notification after all servers finish (parent_deployment_uuid).
- Block persistent volumes on multi-server applications (API, Livewire, SQLite connect)
  and warn in the deployment log for existing applications that have both.
- Remove dead code in push_to_docker_registry().

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras BacsaiandClaude Opus 5.5 committed 2026-09-28 15:38:57 +02:00
1 parent 1f0e2f74d0
commit 7bd8baa7f0
19 files changed
+813 -59

No files matched your search

+1
View File
@@ -24,6 +24,7 @@
- A source-text check is acceptable only for a correctness or security rule that rendering cannot prove easily, such as stable `wire:key` values or no raw user output.
- Call the app code under test. Do not copy app logic into a test, assert only literals, constants, `class_exists`/`method_exists`, or PHP built-ins such as `escapeshellarg`.
- Pest test files share one global function scope. Give file-level helper functions a unique name.
- Call `Server::flushIdentityMap()` in `beforeEach` of tests that use `Server` models; the identity map leaks servers and settings between test files in one run.
## Keep dynamic Livewire identities stable
- In dynamic lists, key components and actions with immutable record identities, not counts, indexes, or array positions.
+8 -2
View File
@@ -20,10 +20,13 @@ class StopApplication
if ($application?->additional_servers?->count() > 0) {
$servers = $servers->merge($application->additional_servers);
}
$errors = [];
foreach ($servers as $server) {
try {
if (! $server->isFunctional()) {
return 'Server is not functional';
$errors[] = "Server {$server->name} is not functional.";
continue;
}
if ($server->isSwarm()) {
@@ -58,9 +61,12 @@ class StopApplication
CleanupDocker::dispatch($server, false, false);
}
} catch (\Exception $e) {
return $e->getMessage();
$errors[] = $e->getMessage();
}
}
if ($errors !== []) {
return implode(' ', $errors);
}
$status = [
'status' => 'exited',
@@ -66,7 +66,7 @@ class EnsureContentFilesOnServer
$written = 0;
foreach ($toWrite as [$fileStorage, $path]) {
try {
$fileStorage->saveStorageOnServer();
$fileStorage->saveStorageOnServer($server);
$written++;
} catch (\Throwable $e) {
$log("Warning: Coolify cannot write the configuration file {$path}: ".$this->plainText($e->getMessage()), 'stderr');
@@ -5287,6 +5287,10 @@ class ApplicationsController extends Controller
], 422);
}
if ($reason = $application->persistentStorageUnavailableReason()) {
return response()->json(['message' => $reason], 422);
}
$storage = LocalPersistentVolume::create([
'name' => $application->uuid.'-'.$request->name,
'mount_path' => $request->mount_path,
@@ -6178,6 +6182,10 @@ class ApplicationsController extends Controller
return response()->json(['message' => 'A destination on this server is already attached.'], 422);
}
if ($reason = $application->additionalServersUnavailableReason()) {
return response()->json(['message' => $reason], 422);
}
$application->additional_networks()->attach($destination->id, ['server_id' => $destination->server_id]);
auditLog('api.application.destination_added', [
+146 -32
View File
@@ -34,6 +34,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Sleep;
use Illuminate\Support\Str;
use JsonException;
@@ -315,6 +316,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
try {
$this->validateDeploymentEnvironmentVariableKeys();
$this->ensureRegistryImageForMultipleServers();
$this->warnAboutVolumesOnMultipleServers();
} catch (Exception $e) {
$this->fail($e);
throw $e;
@@ -619,6 +622,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
);
$this->generate_image_names();
$this->generate_compose_file();
if ($this->pull_image_for_additional_server()) {
$this->save_runtime_environment_variables();
$this->rolling_update();
return;
}
// Save build-time .env file BEFORE the build
$this->save_buildtime_environment_variables();
@@ -635,6 +644,26 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->rolling_update();
}
/**
* An inline Dockerfile image has no commit tag. The main server builds it and pushes it with the
* `latest` tag just before it queues the additional servers, so an additional server pulls that
* image instead of building its own. A failed pull fails the deployment.
*/
private function pull_image_for_additional_server(): bool
{
if (! $this->is_this_additional_server || str($this->application->docker_registry_image_name)->isEmpty()) {
return false;
}
$this->application_deployment_queue->addLogEntry("Pulling image ({$this->production_image_name}) that the main server pushed. Build step skipped.");
$this->execute_remote_command([
'docker pull '.escapeshellarg($this->production_image_name),
'hidden' => true,
]);
return true;
}
private function deploy_dockerimage_buildpack()
{
$this->dockerImage = $this->application->docker_registry_image_name;
@@ -715,14 +744,14 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$fileStorage->is_directory = true;
$fileStorage->content = null;
$fileStorage->save();
$fileStorage->deleteStorageOnServer();
$fileStorage->saveStorageOnServer();
$fileStorage->deleteStorageOnServer($this->mainServer);
$fileStorage->saveStorageOnServer($this->mainServer);
} elseif ($fileStat->value() === 'regular file' && $fileStorage->is_directory) {
$fileStorage->is_directory = false;
$fileStorage->is_based_on_git = true;
$fileStorage->save();
$fileStorage->deleteStorageOnServer();
$fileStorage->saveStorageOnServer();
$fileStorage->deleteStorageOnServer($this->mainServer);
$fileStorage->saveStorageOnServer($this->mainServer);
}
}
}
@@ -1002,16 +1031,13 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
/**
* The queued ServerStorageSaveJob writes new content files, but it can run after the containers
* start. Preserve-repository deployments write all file storages in write_deployment_configurations().
* Coolify writes file storages only on the main server of the application.
* Each deployment writes the files on its own server, so additional servers also get them.
*/
private function write_missing_content_files(): void
{
if ($this->preserveRepository) {
return;
}
if ((int) data_get($this->application, 'destination.server_id') !== (int) $this->mainServer->id) {
return;
}
EnsureContentFilesOnServer::run(
$this->application->fileStorages()->get(),
@@ -1173,7 +1199,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
foreach ($this->application->fileStorages as $fileStorage) {
if (! $fileStorage->is_host_file && ! $fileStorage->is_based_on_git && ! $fileStorage->is_directory) {
$fileStorage->saveStorageOnServer();
$fileStorage->saveStorageOnServer($this->mainServer);
}
}
if ($this->use_build_server) {
@@ -1216,7 +1242,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function push_to_docker_registry()
{
$forceFail = true;
if (str($this->application->docker_registry_image_name)->isEmpty()) {
return;
}
@@ -1226,15 +1251,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
if ($this->application->build_pack === 'dockerimage') {
return;
}
if ($this->use_build_server) {
$forceFail = true;
}
if ($this->server->isSwarm() && $this->build_pack !== 'dockerimage') {
$forceFail = true;
}
if ($this->application->additional_servers->count() > 0) {
$forceFail = true;
}
if ($this->is_this_additional_server) {
return;
}
@@ -1266,9 +1282,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
} catch (Exception $e) {
$this->application_deployment_queue->addLogEntry('Failed to push image to docker registry. Please check debug logs for more information.');
if ($forceFail) {
throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
}
throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
}
}
@@ -2139,6 +2153,42 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
}
/**
* Coolify blocks new volumes on multi-server applications, but older applications can have both.
* Some applications accept separate data on each server (for example a cache), so the
* deployment continues and only shows a warning.
*/
private function warnAboutVolumesOnMultipleServers(): void
{
if ($this->pull_request_id !== 0) {
return;
}
if ($this->application->additional_servers()->doesntExist() || $this->application->persistentStorages()->doesntExist()) {
return;
}
$this->application_deployment_queue->addLogEntry('Warning: This application uses multiple servers and has persistent storage. Volumes are not shared between servers, so each server has its own data.', 'stderr');
}
/**
* Additional servers pull the image that the main server pushes. Without a registry image, each
* server builds its own image, so the servers can run different code.
*/
private function ensureRegistryImageForMultipleServers(): void
{
if ($this->pull_request_id !== 0) {
return;
}
if ($this->application->additional_servers()->doesntExist()) {
return;
}
if (str($this->application->docker_registry_image_name)->isNotEmpty()) {
return;
}
throw new DeploymentException('Before deploying to multiple servers, you must set a Docker image in the General tab. More information: https://coolify.io/docs/knowledge-base/server/multiple-servers');
}
/**
* Build-time names go into shell and Docker build commands, so they must be valid when the
* deployment builds an image. Runtime-only variables only go into the .env file: existing ones
@@ -2640,23 +2690,27 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->prepare_builder_image(firstTry: false);
}
private function deploy_to_additional_destinations()
/**
* @return int The number of deployments that Coolify queued on additional servers
*/
private function deploy_to_additional_destinations(): int
{
if ($this->application->additional_networks->count() === 0) {
return;
return 0;
}
if ($this->pull_request_id !== 0) {
return;
return 0;
}
$destination_ids = $this->application->additional_networks->pluck('id');
if ($this->server->isSwarm()) {
$this->application_deployment_queue->addLogEntry('Additional destinations are not supported in swarm mode.');
return;
return 0;
}
if ($destination_ids->contains($this->destination->id)) {
return;
return 0;
}
$queued = 0;
foreach ($destination_ids as $destination_id) {
$destination = StandaloneDocker::find($destination_id);
if (! $destination) {
@@ -2669,16 +2723,29 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
continue;
}
$deployment_uuid = new_public_id();
queue_application_deployment(
// Deploy the exact commit and image tag of the main server, also for a rollback.
$result = queue_application_deployment(
deployment_uuid: $deployment_uuid,
application: $this->application,
commit: $this->commit,
no_questions_asked: true,
server: $server,
destination: $destination,
no_questions_asked: true,
rollback: $this->rollback,
docker_registry_image_tag: $this->application_deployment_queue->docker_registry_image_tag,
parent_deployment_uuid: $this->deployment_uuid,
);
if ($result['status'] !== 'queued') {
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name} was not queued: {$result['message']}", 'stderr');
continue;
}
$queued++;
$deployment_url = base_url().'/project/'.data_get($this->application, 'environment.project.uuid').'/environment/'.data_get($this->application, 'environment.uuid').'/application/'.data_get($this->application, 'uuid')."/deployment/{$deployment_uuid}";
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: {$deployment_url}");
}
return $queued;
}
private function set_coolify_variables()
@@ -5547,11 +5614,20 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
event(new ApplicationConfigurationChanged($this->application->team()->id));
if (! $this->only_this_server) {
$this->deploy_to_additional_destinations();
if (filled($this->application_deployment_queue->parent_deployment_uuid)) {
$this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);
return;
}
$this->sendDeploymentNotification(DeploymentSuccess::class);
$queuedAdditionalDeployments = $this->only_this_server ? 0 : $this->deploy_to_additional_destinations();
if ($queuedAdditionalDeployments === 0) {
$this->sendDeploymentNotification(DeploymentSuccess::class);
return;
}
Cache::put("multi-server-deployment-queued:{$this->deployment_uuid}", true, now()->addDay());
$this->sendMultiServerDeploymentNotification($this->deployment_uuid);
}
/**
@@ -5559,9 +5635,47 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
*/
private function handleFailedDeployment(): void
{
if (filled($this->application_deployment_queue->parent_deployment_uuid)) {
$this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);
return;
}
$this->sendDeploymentNotification(DeploymentFailed::class);
}
/**
* The main server queues one deployment for each additional server. When all are queued and the
* last one ends, Coolify sends one notification: success if all servers succeeded, else failed
* with a link to the first deployment that did not succeed.
*/
private function sendMultiServerDeploymentNotification(string $parentDeploymentUuid): void
{
if (! Cache::has("multi-server-deployment-queued:{$parentDeploymentUuid}")) {
return;
}
$deployments = ApplicationDeploymentQueue::query()
->where('parent_deployment_uuid', $parentDeploymentUuid)
->get(['deployment_uuid', 'status']);
$endStatuses = [
ApplicationDeploymentStatus::FINISHED->value,
ApplicationDeploymentStatus::FAILED->value,
ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
];
if ($deployments->contains(fn (ApplicationDeploymentQueue $deployment) => ! in_array($deployment->status, $endStatuses, true))) {
return;
}
if (! Cache::add("multi-server-deployment-notified:{$parentDeploymentUuid}", true, now()->addDay())) {
return;
}
$unsuccessful = $deployments->first(fn (ApplicationDeploymentQueue $deployment) => $deployment->status !== ApplicationDeploymentStatus::FINISHED->value);
$this->application->environment->project->team?->notify($unsuccessful
? new DeploymentFailed($this->application, $unsuccessful->deployment_uuid, $this->preview)
: new DeploymentSuccess($this->application, $parentDeploymentUuid, $this->preview));
}
/**
* Send deployment status notification to the team.
*/
@@ -97,6 +97,10 @@ class ConnectApplication extends Component
return null;
}
if ($reason = $application->persistentStorageUnavailableReason()) {
throw new \Exception($reason);
}
if ($application->persistentStorages()->where('standalone_sqlite_id', $this->database->id)->exists()) {
throw new \Exception("{$application->name} already mounts this database volume.");
}
+14
View File
@@ -240,6 +240,14 @@ class Storage extends Component
return collect($this->fileStorage)->where('is_directory', true);
}
/**
* The reason why the resource cannot get a volume mount, or null if it can.
*/
public function getVolumeUnavailableReasonProperty(): ?string
{
return $this->resource instanceof Application ? $this->resource->persistentStorageUnavailableReason() : null;
}
public function getVolumeCountProperty()
{
return $this->cachedVolumeCount;
@@ -265,6 +273,12 @@ class Storage extends Component
'mount_path' => 'required|string',
], ValidationPatterns::volumeNameMessages());
if ($reason = $this->volumeUnavailableReason) {
$this->dispatch('error', 'Failed to add volume.', $reason);
return;
}
$name = $this->resource->uuid.'-'.$this->name;
LocalPersistentVolume::create([
@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Shared;
use App\Actions\Application\StopApplicationOneServer;
use App\Actions\Docker\GetContainersStatus;
use App\Events\ApplicationStatusChanged;
use App\Models\Application;
use App\Models\Server;
use App\Models\StandaloneDocker;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
@@ -156,6 +157,14 @@ class Destination extends Component
$server = Server::ownedByCurrentTeam()->findOrFail($server_id);
$network = StandaloneDocker::ownedByCurrentTeam()->where('server_id', $server->id)->findOrFail($network_id);
$this->authorize('update', $this->resource);
$reason = $this->resource instanceof Application
? $this->resource->additionalServersUnavailableReason()
: 'Only applications can use multiple servers.';
if ($reason) {
$this->dispatch('error', 'Failed to add server.', $reason);
return;
}
$this->resource->additional_networks()->syncWithoutDetaching([
$network->id => ['server_id' => $server->id],
+27
View File
@@ -585,6 +585,33 @@ class Application extends BaseModel
->withPivot('server_id', 'status');
}
/**
* The reason why this application cannot get a persistent volume, or null if it can.
*/
public function persistentStorageUnavailableReason(): ?string
{
if ($this->additional_servers()->exists()) {
return 'Applications that use multiple servers cannot have persistent storage because volumes are not shared between servers.';
}
return null;
}
/**
* The reason why this application cannot deploy to additional servers, or null if it can.
*/
public function additionalServersUnavailableReason(): ?string
{
if ($this->build_pack === 'dockercompose') {
return 'Docker Compose applications cannot use multiple servers.';
}
if ($this->persistentStorages()->exists()) {
return 'Applications with persistent storage cannot use multiple servers because volumes are not shared between servers.';
}
return null;
}
public function is_public_repository(): bool
{
if (data_get($this, 'source.is_public')) {
@@ -39,6 +39,7 @@ use OpenApi\Attributes as OA;
'deployment_url' => ['type' => 'string'],
'destination_id' => ['type' => 'string'],
'only_this_server' => ['type' => 'boolean'],
'parent_deployment_uuid' => ['type' => 'string', 'nullable' => true],
'rollback' => ['type' => 'boolean'],
'commit_message' => ['type' => 'string'],
],
@@ -105,6 +106,7 @@ class ApplicationDeploymentQueue extends Model
'deployment_url',
'destination_id',
'only_this_server',
'parent_deployment_uuid',
'rollback',
'commit_message',
'is_api',
+61 -16
View File
@@ -4,10 +4,12 @@ namespace App\Models;
use App\Events\FileStorageChanged;
use App\Jobs\ServerStorageSaveJob;
use Closure;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Database\Eloquent\Relations\MorphTo;
use Illuminate\Support\Collection;
use Symfony\Component\Yaml\Yaml;
class LocalFileVolume extends BaseModel
@@ -236,21 +238,21 @@ class LocalFileVolume extends BaseModel
return (string) $content;
}
public function deleteStorageOnServer()
/**
* Without a server, Coolify deletes the file on every server of the resource.
*/
public function deleteStorageOnServer(?Server $server = null)
{
if ($this->is_host_file) {
return;
}
if (is_null($server)) {
return $this->runOnEveryServer(fn (Server $server) => $this->deleteStorageOnServer($server));
}
$this->load(['service']);
$isService = data_get($this->resource, 'service');
if ($isService) {
$workdir = $this->resource->service->workdir();
$server = $this->resource->service->server;
} else {
$workdir = $this->resource->workdir();
$server = $this->resource->destination->server;
}
$workdir = $isService ? $this->resource->service->workdir() : $this->resource->workdir();
$commands = collect([]);
$path = data_get_str($this, 'fs_path');
if ($path->startsWith('.')) {
@@ -282,21 +284,21 @@ class LocalFileVolume extends BaseModel
}
}
public function saveStorageOnServer()
/**
* Without a server, Coolify writes the file on every server of the resource.
*/
public function saveStorageOnServer(?Server $server = null)
{
if ($this->is_host_file) {
return;
}
if (is_null($server)) {
return $this->runOnEveryServer(fn (Server $server) => $this->saveStorageOnServer($server));
}
$this->load(['service']);
$isService = data_get($this->resource, 'service');
if ($isService) {
$workdir = $this->resource->service->workdir();
$server = $this->resource->service->server;
} else {
$workdir = $this->resource->workdir();
$server = $this->resource->destination->server;
}
$workdir = $isService ? $this->resource->service->workdir() : $this->resource->workdir();
$commands = collect([]);
$escapedWorkdir = escapeshellarg($workdir);
@@ -555,6 +557,49 @@ class LocalFileVolume extends BaseModel
: $owner->workdir();
}
/**
* The main server of the resource, and the additional servers of an application.
*
* @return Collection<int, Server>
*/
public function servers(): Collection
{
$this->load(['service']);
if (data_get($this->resource, 'service')) {
return collect([$this->resource->service->server]);
}
$servers = collect([$this->resource->destination->server]);
if ($this->resource instanceof Application) {
$servers = $servers->merge($this->resource->additional_servers);
}
return $servers->filter()->unique('id')->values();
}
/**
* Runs the callback on every server, also when one server fails, and then throws the first error.
*
* @param Closure(Server): mixed $callback
*/
protected function runOnEveryServer(Closure $callback): mixed
{
$result = null;
$firstError = null;
foreach ($this->servers() as $server) {
try {
$result = $callback($server);
} catch (\Throwable $e) {
$firstError ??= $e;
}
}
if ($firstError) {
throw $firstError;
}
return $result;
}
/**
* The Application, Service or standalone database that owns the storage directory.
*/
+3 -2
View File
@@ -12,7 +12,7 @@ use App\Models\StandaloneDocker;
use Illuminate\Support\Facades\DB;
use Spatie\Url\Url;
function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null)
function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null, ?string $parent_deployment_uuid = null)
{
$commit = $commit ?: ($application->git_commit_sha ?: 'HEAD');
$commit = validateGitRef($commit, 'deployment commit');
@@ -31,7 +31,7 @@ function queue_application_deployment(Application $application, string $deployme
$destination_id = $destination->id;
}
$admission = DB::transaction(function () use ($application, $application_id, $commit, $deployment_uuid, $deployment_url, $destination_id, $docker_registry_image_tag, $force_rebuild, $git_type, $is_api, $is_webhook, $no_questions_asked, $only_this_server, $pull_request_id, $restart_only, $rollback, $server_id, $server_name) {
$admission = DB::transaction(function () use ($application, $application_id, $commit, $deployment_uuid, $deployment_url, $destination_id, $docker_registry_image_tag, $force_rebuild, $git_type, $is_api, $is_webhook, $no_questions_asked, $only_this_server, $parent_deployment_uuid, $pull_request_id, $restart_only, $rollback, $server_id, $server_name) {
// Lock stable rows because an empty deployment queue has no row to lock.
Application::query()->whereKey($application_id)->lockForUpdate()->firstOrFail();
$serverForQueueCheck = Server::query()->whereKey($server_id)->lockForUpdate()->firstOrFail();
@@ -81,6 +81,7 @@ function queue_application_deployment(Application $application, string $deployme
'rollback' => $rollback,
'git_type' => $git_type,
'only_this_server' => $only_this_server,
'parent_deployment_uuid' => $parent_deployment_uuid,
]);
});
@@ -0,0 +1,27 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Links the deployments on additional servers to the main deployment, so Coolify can send one
* notification after all servers finish.
*/
public function up(): void
{
Schema::table('application_deployment_queues', function (Blueprint $table) {
$table->string('parent_deployment_uuid')->nullable()->index();
});
}
public function down(): void
{
Schema::table('application_deployment_queues', function (Blueprint $table) {
$table->dropIndex(['parent_deployment_uuid']);
$table->dropColumn('parent_deployment_uuid');
});
}
};
+4
View File
@@ -23241,6 +23241,10 @@
"only_this_server": {
"type": "boolean"
},
"parent_deployment_uuid": {
"type": "string",
"nullable": true
},
"rollback": {
"type": "boolean"
},
+3
View File
@@ -14873,6 +14873,9 @@ components:
type: string
only_this_server:
type: boolean
parent_deployment_uuid:
type: string
nullable: true
rollback:
type: boolean
commit_message:
@@ -112,6 +112,11 @@
})
}
})">
@if ($this->volumeUnavailableReason)
<x-callout type="warning" title="Volume mounts are unavailable" class="w-full">
{{ $this->volumeUnavailableReason }}
</x-callout>
@else
<form class="flex w-full flex-col gap-4"
wire:submit='submitPersistentVolume'>
<p class="text-[13px] leading-5 text-neutral-500 dark:text-fg-dim">
@@ -130,6 +135,7 @@
</div>
</div>
</form>
@endif
</div>
</div>
</div>
+64
View File
@@ -482,6 +482,70 @@ describe('Application multi-destination cross-team', function () {
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
});
test('attaches a destination on another server of the same team', function () {
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
$this->withHeaders($this->headers)
->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
'destination_uuid' => $secondDestination->uuid,
])
->assertSuccessful();
expect($this->application->fresh()->additional_networks)->toHaveCount(1);
});
test('rejects an additional server for an application with persistent storage', function () {
LocalPersistentVolume::create([
'name' => 'app-data-'.$this->application->uuid,
'mount_path' => '/data',
'resource_id' => $this->application->id,
'resource_type' => $this->application->getMorphClass(),
]);
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
$this->withHeaders($this->headers)
->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
'destination_uuid' => $secondDestination->uuid,
])
->assertUnprocessable()
->assertJsonPath('message', 'Applications with persistent storage cannot use multiple servers because volumes are not shared between servers.');
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
});
test('rejects an additional server for a Docker Compose application', function () {
$this->application->update(['build_pack' => 'dockercompose']);
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
$this->withHeaders($this->headers)
->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
'destination_uuid' => $secondDestination->uuid,
])
->assertUnprocessable();
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
});
test('rejects a volume for an application with an additional server', function () {
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
$this->application->additional_networks()->attach($secondDestination->id, ['server_id' => $secondServer->id]);
$this->withHeaders($this->headers)
->postJson("/api/v1/applications/{$this->application->uuid}/storages", [
'type' => 'persistent',
'name' => 'data',
'mount_path' => '/data',
])
->assertUnprocessable()
->assertJsonPath('message', 'Applications that use multiple servers cannot have persistent storage because volumes are not shared between servers.');
expect($this->application->persistentStorages()->count())->toBe(0);
});
test('lists primary destination', function () {
$response = $this->withHeaders($this->headers)
->getJson("/api/v1/applications/{$this->application->uuid}/destinations");
@@ -200,8 +200,8 @@ function runComposeStart(object $test, array $properties = []): ContentFilesUpDe
foreach ([
'application' => $test->application->fresh(),
'application_deployment_queue' => $queue,
'server' => $test->server,
'mainServer' => $test->server,
'server' => $test->server->fresh(),
'mainServer' => $test->server->fresh(),
'deployment_uuid' => 'deployment-uuid',
'workdir' => '/artifacts/deployment-uuid',
'configuration_dir' => $test->application->workdir(),
@@ -343,18 +343,22 @@ test('a preserve-repository deployment writes content files before docker compos
->and($write)->toBeLessThan(ContentFilesUpRecorder::indexOf('deployment', ' up -d'));
});
test('a deployment to another server does not check the content files of the main server', function () {
test('a deployment to an additional server writes missing content files on that server', function () {
$volume = contentFilesVolume($this->application, 'config/app.conf');
$otherServer = Server::factory()->create([
'team_id' => $this->team->id,
'private_key_id' => $this->server->private_key_id,
]);
])->fresh();
fakeContentFilesServer([$volume->fs_path => 'missing']);
runComposeStart($this, ['server' => $otherServer, 'mainServer' => $otherServer]);
expect(contentFilesStateChecks())->toBe([]);
contentFilesAssertNoWrite();
$write = ContentFilesUpRecorder::indexOf('ssh', 'base64 -d | tee '.escapeshellarg($volume->fs_path));
expect(contentFilesStateChecks())->toHaveCount(1)
->and(contentFilesStateChecks()[0])->toContain("@'{$otherServer->ip}'")
->and($write)->not->toBeNull()
->and(ContentFilesUpRecorder::ssh()[$write])->toContain("@'{$otherServer->ip}'")
->and(ContentFilesUpRecorder::ssh()[$write])->not->toContain("@'{$this->server->ip}'");
});
test('the content file check and write are safe for non-root servers', function () {
@@ -0,0 +1,415 @@
<?php
use App\Actions\Application\StopApplication;
use App\Enums\ApplicationDeploymentStatus;
use App\Exceptions\DeploymentException;
use App\Jobs\ApplicationDeploymentJob;
use App\Livewire\Project\Database\Sqlite\ConnectApplication;
use App\Livewire\Project\Service\Storage;
use App\Livewire\Project\Shared\Destination;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\LocalFileVolume;
use App\Models\LocalPersistentVolume;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandaloneSqlite;
use App\Models\Team;
use App\Models\User;
use App\Notifications\Application\DeploymentFailed;
use App\Notifications\Application\DeploymentSuccess;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Queue;
use Livewire\Livewire;
uses(RefreshDatabase::class);
beforeEach(function () {
Server::flushIdentityMap();
Queue::fake();
Notification::fake();
config()->set('cache.default', 'array');
InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
$this->team = Team::factory()->create();
$this->team->emailNotificationSettings->update([
'smtp_enabled' => true,
'deployment_success_email_notifications' => true,
'deployment_failure_email_notifications' => true,
]);
$this->user = User::factory()->create();
$this->user->teams()->attach($this->team, ['role' => 'owner']);
$this->mainServer = Server::factory()->create(['team_id' => $this->team->id, 'name' => 'main-server']);
$this->mainDestination = StandaloneDocker::where('server_id', $this->mainServer->id)->firstOrFail();
$this->extraServer = Server::factory()->create(['team_id' => $this->team->id, 'name' => 'extra-server']);
$this->extraDestination = StandaloneDocker::where('server_id', $this->extraServer->id)->firstOrFail();
$project = Project::factory()->create(['team_id' => $this->team->id]);
$environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]);
$this->application = Application::factory()->create([
'environment_id' => $environment->id,
'destination_id' => $this->mainDestination->id,
'destination_type' => $this->mainDestination->getMorphClass(),
'build_pack' => 'dockerfile',
]);
$this->application->additional_networks()->attach($this->extraDestination->id, ['server_id' => $this->extraServer->id]);
$this->application->refresh();
});
/**
* @param array<string, mixed> $properties
*/
function makeMultiServerDeploymentJob(array $properties): array
{
$reflection = new ReflectionClass(ApplicationDeploymentJob::class);
$job = $reflection->newInstanceWithoutConstructor();
foreach (['pull_request_id' => 0, 'preview' => null, 'only_this_server' => false, 'commit' => 'HEAD', 'rollback' => false, ...$properties] as $name => $value) {
$reflection->getProperty($name)->setValue($job, $value);
}
return [$job, $reflection];
}
function callMultiServerDeploymentJob(array $jobAndReflection, string $method, mixed ...$arguments): mixed
{
[$job, $reflection] = $jobAndReflection;
return $reflection->getMethod($method)->invoke($job, ...$arguments);
}
function createMultiServerDeployment(Application $application, Server $server, string $status, ?string $parentDeploymentUuid = null): ApplicationDeploymentQueue
{
return ApplicationDeploymentQueue::create([
'application_id' => $application->id,
'deployment_uuid' => new_public_id(),
'server_id' => $server->id,
'status' => $status,
'parent_deployment_uuid' => $parentDeploymentUuid,
]);
}
describe('registry image guard', function () {
test('fails a multi-server deployment without a registry image', function () {
$job = makeMultiServerDeploymentJob(['application' => $this->application]);
expect(fn () => callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers'))
->toThrow(DeploymentException::class, 'Before deploying to multiple servers');
});
test('allows a multi-server deployment with a registry image', function () {
$this->application->update(['docker_registry_image_name' => 'registry.example.com/app']);
$job = makeMultiServerDeploymentJob(['application' => $this->application->fresh()]);
callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
})->throwsNoExceptions();
test('allows preview deployments, which run only on the main server', function () {
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'pull_request_id' => 7]);
callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
})->throwsNoExceptions();
});
function makeVolumeWarningJob(Application $application, array &$logEntries): array
{
$queue = Mockery::mock(ApplicationDeploymentQueue::class);
$queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message, string $type = 'stdout') use (&$logEntries) {
$logEntries[] = [$message, $type];
});
return makeMultiServerDeploymentJob(['application' => $application, 'application_deployment_queue' => $queue]);
}
describe('volume warning', function () {
test('warns in the deployment log when an application with a volume uses multiple servers', function () {
LocalPersistentVolume::create([
'name' => 'app-data-'.$this->application->uuid,
'mount_path' => '/data',
'resource_id' => $this->application->id,
'resource_type' => $this->application->getMorphClass(),
]);
$logEntries = [];
$job = makeVolumeWarningJob($this->application->fresh(), $logEntries);
callMultiServerDeploymentJob($job, 'warnAboutVolumesOnMultipleServers');
expect($logEntries)->toBe([[
'Warning: This application uses multiple servers and has persistent storage. Volumes are not shared between servers, so each server has its own data.',
'stderr',
]]);
});
test('does not warn when the application has no volume', function () {
$logEntries = [];
$job = makeVolumeWarningJob($this->application, $logEntries);
callMultiServerDeploymentJob($job, 'warnAboutVolumesOnMultipleServers');
expect($logEntries)->toBe([]);
});
});
describe('combined deployment notification', function () {
test('links queued additional deployments to the main deployment', function () {
$parent = createMultiServerDeployment($this->application, $this->mainServer, ApplicationDeploymentStatus::FINISHED->value);
$job = makeMultiServerDeploymentJob([
'application' => $this->application,
'application_deployment_queue' => $parent,
'deployment_uuid' => $parent->deployment_uuid,
'server' => $this->mainServer,
'mainServer' => $this->mainServer,
'destination' => $this->mainDestination,
]);
expect(callMultiServerDeploymentJob($job, 'deploy_to_additional_destinations'))->toBe(1);
$child = ApplicationDeploymentQueue::where('server_id', $this->extraServer->id)->sole();
expect($child->parent_deployment_uuid)->toBe($parent->deployment_uuid);
});
test('additional servers deploy the same commit and image tag as a rollback on the main server', function () {
$parent = ApplicationDeploymentQueue::create([
'application_id' => $this->application->id,
'deployment_uuid' => new_public_id(),
'server_id' => $this->mainServer->id,
'status' => ApplicationDeploymentStatus::FINISHED->value,
'commit' => 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678',
'rollback' => true,
'docker_registry_image_tag' => 'v1.2.3',
]);
$job = makeMultiServerDeploymentJob([
'application' => $this->application,
'application_deployment_queue' => $parent,
'deployment_uuid' => $parent->deployment_uuid,
'server' => $this->mainServer,
'mainServer' => $this->mainServer,
'destination' => $this->mainDestination,
'commit' => $parent->commit,
'rollback' => true,
]);
callMultiServerDeploymentJob($job, 'deploy_to_additional_destinations');
$child = ApplicationDeploymentQueue::where('server_id', $this->extraServer->id)->sole();
expect($child->commit)->toBe('a1b2c3d4e5f60718293a4b5c6d7e8f9012345678')
->and((bool) $child->rollback)->toBeTrue()
->and($child->docker_registry_image_tag)->toBe('v1.2.3');
});
test('does not notify while an additional server is still deploying', function () {
$parentUuid = new_public_id();
Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
$failed = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FAILED->value, $parentUuid);
createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::IN_PROGRESS->value, $parentUuid);
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $failed]);
callMultiServerDeploymentJob($job, 'handleFailedDeployment');
Notification::assertNothingSent();
});
test('does not notify before the main deployment queued all additional servers', function () {
$parentUuid = new_public_id();
$finished = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $finished]);
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
Notification::assertNothingSent();
});
test('sends one success notification after all additional servers succeed', function () {
$parentUuid = new_public_id();
Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
$first = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $first]);
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
Notification::assertSentToTimes($this->team, DeploymentSuccess::class, 1);
Notification::assertSentTo($this->team, DeploymentSuccess::class, fn (DeploymentSuccess $notification) => $notification->deployment_uuid === $parentUuid);
Notification::assertNotSentTo($this->team, DeploymentFailed::class);
});
test('sends a failed notification that links to the failed server deployment', function () {
$parentUuid = new_public_id();
Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
$finished = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
$failed = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FAILED->value, $parentUuid);
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $finished]);
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
Notification::assertSentTo($this->team, DeploymentFailed::class, fn (DeploymentFailed $notification) => $notification->deployment_uuid === $failed->deployment_uuid);
Notification::assertNotSentTo($this->team, DeploymentSuccess::class);
});
});
/**
* Records the deployment commands instead of running them on a server.
*/
class MultiServerRecordingDeploymentJob extends ApplicationDeploymentJob
{
/** @var list<string> */
public array $recordedCommands = [];
public function __construct() {}
public function execute_remote_command(...$commands): void
{
foreach ($commands as $command) {
$this->recordedCommands[] = $command['command'] ?? $command[0];
}
}
}
function makeInlineDockerfileJob(Application $application, bool $isAdditionalServer): array
{
$reflection = new ReflectionClass(ApplicationDeploymentJob::class);
$job = new MultiServerRecordingDeploymentJob;
$queue = Mockery::mock(ApplicationDeploymentQueue::class);
$queue->shouldReceive('addLogEntry');
foreach ([
'application' => $application,
'application_deployment_queue' => $queue,
'is_this_additional_server' => $isAdditionalServer,
'pull_request_id' => 0,
] as $name => $value) {
$reflection->getProperty($name)->setValue($job, $value);
}
$reflection->getMethod('generate_image_names')->invoke($job);
return [$job, $reflection];
}
describe('inline Dockerfile image on additional servers', function () {
beforeEach(function () {
$this->application->update([
'dockerfile' => "FROM nginx:alpine\n",
'docker_registry_image_name' => 'registry.example.com/nginx-multi',
]);
});
test('an additional server pulls the image that the main server pushed', function () {
[$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), true);
expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeTrue()
->and($job->recordedCommands)->toBe(["docker pull 'registry.example.com/nginx-multi:latest'"]);
});
test('the main server builds the image', function () {
[$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), false);
expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeFalse()
->and($job->recordedCommands)->toBe([]);
});
});
test('stopping continues on the other servers when one server is not functional', function () {
$this->mainServer->settings->update(['is_reachable' => false]);
$this->extraServer->settings->update(['is_reachable' => false]);
$result = StopApplication::run($this->application->fresh());
expect($result)->toContain('main-server')->toContain('extra-server');
});
test('file storages of an application belong to all of its servers', function () {
$file = LocalFileVolume::create([
'fs_path' => '/data/coolify/applications/'.$this->application->uuid.'/config.yml',
'mount_path' => '/app/config.yml',
'content' => 'key: value',
'resource_id' => $this->application->id,
'resource_type' => $this->application->getMorphClass(),
]);
expect($file->servers()->pluck('id')->all())->toBe([$this->mainServer->id, $this->extraServer->id]);
});
describe('adding a server in the UI', function () {
beforeEach(function () {
$this->application->additional_networks()->detach();
$this->actingAs($this->user);
session(['currentTeam' => $this->team]);
});
test('rejects an application with persistent storage', function () {
LocalPersistentVolume::create([
'name' => 'app-data-'.$this->application->uuid,
'mount_path' => '/data',
'resource_id' => $this->application->id,
'resource_type' => $this->application->getMorphClass(),
]);
Livewire::test(Destination::class, ['resource' => $this->application->fresh()])
->call('addServer', $this->extraDestination->id, $this->extraServer->id)
->assertDispatched('error');
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
});
test('adds a server to an application without persistent storage', function () {
Livewire::test(Destination::class, ['resource' => $this->application->fresh()])
->call('addServer', $this->extraDestination->id, $this->extraServer->id);
expect($this->application->fresh()->additional_networks)->toHaveCount(1);
});
});
describe('adding a volume to an application with additional servers', function () {
beforeEach(function () {
$this->actingAs($this->user);
session(['currentTeam' => $this->team]);
});
test('the storage page rejects the volume', function () {
Livewire::test(Storage::class, ['resource' => $this->application->fresh()])
->assertSee('Volume mounts are unavailable')
->set('name', 'data')
->set('mount_path', '/data')
->call('submitPersistentVolume')
->assertDispatched('error');
expect($this->application->persistentStorages()->count())->toBe(0);
});
test('connecting a SQLite database rejects the volume', function () {
$sqlite = StandaloneSqlite::create([
'name' => 'app-sqlite',
'environment_id' => $this->application->environment_id,
'destination_id' => $this->mainDestination->id,
'destination_type' => $this->mainDestination->getMorphClass(),
]);
Livewire::test(ConnectApplication::class, ['database' => $sqlite])
->set('applicationUuid', $this->application->uuid)
->set('mountPath', '/app/database')
->call('connect')
->assertNoRedirect();
expect($this->application->persistentStorages()->count())->toBe(0);
});
test('an application on one server can still add a volume', function () {
$this->application->additional_networks()->detach();
Livewire::test(Storage::class, ['resource' => $this->application->fresh()])
->set('name', 'data')
->set('mount_path', '/data')
->call('submitPersistentVolume')
->assertNotDispatched('error')
->assertDontSee('Volume mounts are unavailable');
expect($this->application->persistentStorages()->count())->toBe(1);
});
});