mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-04 10:42:23 +00:00
fix(multi-server): make applications on several servers consistent
- Stop the application on every server, also when one server is not functional. - Write file mounts on every server of the application, not only on the main server. - Deploy the exact commit, rollback flag and image tag of the main deployment to the additional servers, so a rollback reverts all servers. - Pull the pushed image on additional servers for inline Dockerfile applications instead of building it again on each server. - Fail multi-server deployments without a Docker image name, also from webhooks and the API. - Send one deployment notification after all servers finish (parent_deployment_uuid). - Block persistent volumes on multi-server applications (API, Livewire, SQLite connect) and warn in the deployment log for existing applications that have both. - Remove dead code in push_to_docker_registry(). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1f0e2f74d0
commit
7bd8baa7f0
19 files changed
+813
-59
No files matched your search
@@ -24,6 +24,7 @@
|
||||
- A source-text check is acceptable only for a correctness or security rule that rendering cannot prove easily, such as stable `wire:key` values or no raw user output.
|
||||
- Call the app code under test. Do not copy app logic into a test, assert only literals, constants, `class_exists`/`method_exists`, or PHP built-ins such as `escapeshellarg`.
|
||||
- Pest test files share one global function scope. Give file-level helper functions a unique name.
|
||||
- Call `Server::flushIdentityMap()` in `beforeEach` of tests that use `Server` models; the identity map leaks servers and settings between test files in one run.
|
||||
|
||||
## Keep dynamic Livewire identities stable
|
||||
- In dynamic lists, key components and actions with immutable record identities, not counts, indexes, or array positions.
|
||||
|
||||
@@ -20,10 +20,13 @@ class StopApplication
|
||||
if ($application?->additional_servers?->count() > 0) {
|
||||
$servers = $servers->merge($application->additional_servers);
|
||||
}
|
||||
$errors = [];
|
||||
foreach ($servers as $server) {
|
||||
try {
|
||||
if (! $server->isFunctional()) {
|
||||
return 'Server is not functional';
|
||||
$errors[] = "Server {$server->name} is not functional.";
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($server->isSwarm()) {
|
||||
@@ -58,9 +61,12 @@ class StopApplication
|
||||
CleanupDocker::dispatch($server, false, false);
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
return $e->getMessage();
|
||||
$errors[] = $e->getMessage();
|
||||
}
|
||||
}
|
||||
if ($errors !== []) {
|
||||
return implode(' ', $errors);
|
||||
}
|
||||
|
||||
$status = [
|
||||
'status' => 'exited',
|
||||
|
||||
@@ -66,7 +66,7 @@ class EnsureContentFilesOnServer
|
||||
$written = 0;
|
||||
foreach ($toWrite as [$fileStorage, $path]) {
|
||||
try {
|
||||
$fileStorage->saveStorageOnServer();
|
||||
$fileStorage->saveStorageOnServer($server);
|
||||
$written++;
|
||||
} catch (\Throwable $e) {
|
||||
$log("Warning: Coolify cannot write the configuration file {$path}: ".$this->plainText($e->getMessage()), 'stderr');
|
||||
|
||||
@@ -5287,6 +5287,10 @@ class ApplicationsController extends Controller
|
||||
], 422);
|
||||
}
|
||||
|
||||
if ($reason = $application->persistentStorageUnavailableReason()) {
|
||||
return response()->json(['message' => $reason], 422);
|
||||
}
|
||||
|
||||
$storage = LocalPersistentVolume::create([
|
||||
'name' => $application->uuid.'-'.$request->name,
|
||||
'mount_path' => $request->mount_path,
|
||||
@@ -6178,6 +6182,10 @@ class ApplicationsController extends Controller
|
||||
return response()->json(['message' => 'A destination on this server is already attached.'], 422);
|
||||
}
|
||||
|
||||
if ($reason = $application->additionalServersUnavailableReason()) {
|
||||
return response()->json(['message' => $reason], 422);
|
||||
}
|
||||
|
||||
$application->additional_networks()->attach($destination->id, ['server_id' => $destination->server_id]);
|
||||
|
||||
auditLog('api.application.destination_added', [
|
||||
|
||||
@@ -34,6 +34,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Sleep;
|
||||
use Illuminate\Support\Str;
|
||||
use JsonException;
|
||||
@@ -315,6 +316,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
|
||||
try {
|
||||
$this->validateDeploymentEnvironmentVariableKeys();
|
||||
$this->ensureRegistryImageForMultipleServers();
|
||||
$this->warnAboutVolumesOnMultipleServers();
|
||||
} catch (Exception $e) {
|
||||
$this->fail($e);
|
||||
throw $e;
|
||||
@@ -619,6 +622,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
);
|
||||
$this->generate_image_names();
|
||||
$this->generate_compose_file();
|
||||
if ($this->pull_image_for_additional_server()) {
|
||||
$this->save_runtime_environment_variables();
|
||||
$this->rolling_update();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Save build-time .env file BEFORE the build
|
||||
$this->save_buildtime_environment_variables();
|
||||
@@ -635,6 +644,26 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$this->rolling_update();
|
||||
}
|
||||
|
||||
/**
|
||||
* An inline Dockerfile image has no commit tag. The main server builds it and pushes it with the
|
||||
* `latest` tag just before it queues the additional servers, so an additional server pulls that
|
||||
* image instead of building its own. A failed pull fails the deployment.
|
||||
*/
|
||||
private function pull_image_for_additional_server(): bool
|
||||
{
|
||||
if (! $this->is_this_additional_server || str($this->application->docker_registry_image_name)->isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->application_deployment_queue->addLogEntry("Pulling image ({$this->production_image_name}) that the main server pushed. Build step skipped.");
|
||||
$this->execute_remote_command([
|
||||
'docker pull '.escapeshellarg($this->production_image_name),
|
||||
'hidden' => true,
|
||||
]);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function deploy_dockerimage_buildpack()
|
||||
{
|
||||
$this->dockerImage = $this->application->docker_registry_image_name;
|
||||
@@ -715,14 +744,14 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$fileStorage->is_directory = true;
|
||||
$fileStorage->content = null;
|
||||
$fileStorage->save();
|
||||
$fileStorage->deleteStorageOnServer();
|
||||
$fileStorage->saveStorageOnServer();
|
||||
$fileStorage->deleteStorageOnServer($this->mainServer);
|
||||
$fileStorage->saveStorageOnServer($this->mainServer);
|
||||
} elseif ($fileStat->value() === 'regular file' && $fileStorage->is_directory) {
|
||||
$fileStorage->is_directory = false;
|
||||
$fileStorage->is_based_on_git = true;
|
||||
$fileStorage->save();
|
||||
$fileStorage->deleteStorageOnServer();
|
||||
$fileStorage->saveStorageOnServer();
|
||||
$fileStorage->deleteStorageOnServer($this->mainServer);
|
||||
$fileStorage->saveStorageOnServer($this->mainServer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1002,16 +1031,13 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
/**
|
||||
* The queued ServerStorageSaveJob writes new content files, but it can run after the containers
|
||||
* start. Preserve-repository deployments write all file storages in write_deployment_configurations().
|
||||
* Coolify writes file storages only on the main server of the application.
|
||||
* Each deployment writes the files on its own server, so additional servers also get them.
|
||||
*/
|
||||
private function write_missing_content_files(): void
|
||||
{
|
||||
if ($this->preserveRepository) {
|
||||
return;
|
||||
}
|
||||
if ((int) data_get($this->application, 'destination.server_id') !== (int) $this->mainServer->id) {
|
||||
return;
|
||||
}
|
||||
|
||||
EnsureContentFilesOnServer::run(
|
||||
$this->application->fileStorages()->get(),
|
||||
@@ -1173,7 +1199,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
}
|
||||
foreach ($this->application->fileStorages as $fileStorage) {
|
||||
if (! $fileStorage->is_host_file && ! $fileStorage->is_based_on_git && ! $fileStorage->is_directory) {
|
||||
$fileStorage->saveStorageOnServer();
|
||||
$fileStorage->saveStorageOnServer($this->mainServer);
|
||||
}
|
||||
}
|
||||
if ($this->use_build_server) {
|
||||
@@ -1216,7 +1242,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
|
||||
private function push_to_docker_registry()
|
||||
{
|
||||
$forceFail = true;
|
||||
if (str($this->application->docker_registry_image_name)->isEmpty()) {
|
||||
return;
|
||||
}
|
||||
@@ -1226,15 +1251,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
if ($this->application->build_pack === 'dockerimage') {
|
||||
return;
|
||||
}
|
||||
if ($this->use_build_server) {
|
||||
$forceFail = true;
|
||||
}
|
||||
if ($this->server->isSwarm() && $this->build_pack !== 'dockerimage') {
|
||||
$forceFail = true;
|
||||
}
|
||||
if ($this->application->additional_servers->count() > 0) {
|
||||
$forceFail = true;
|
||||
}
|
||||
if ($this->is_this_additional_server) {
|
||||
return;
|
||||
}
|
||||
@@ -1266,9 +1282,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->application_deployment_queue->addLogEntry('Failed to push image to docker registry. Please check debug logs for more information.');
|
||||
if ($forceFail) {
|
||||
throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
|
||||
}
|
||||
throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2139,6 +2153,42 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Coolify blocks new volumes on multi-server applications, but older applications can have both.
|
||||
* Some applications accept separate data on each server (for example a cache), so the
|
||||
* deployment continues and only shows a warning.
|
||||
*/
|
||||
private function warnAboutVolumesOnMultipleServers(): void
|
||||
{
|
||||
if ($this->pull_request_id !== 0) {
|
||||
return;
|
||||
}
|
||||
if ($this->application->additional_servers()->doesntExist() || $this->application->persistentStorages()->doesntExist()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->application_deployment_queue->addLogEntry('Warning: This application uses multiple servers and has persistent storage. Volumes are not shared between servers, so each server has its own data.', 'stderr');
|
||||
}
|
||||
|
||||
/**
|
||||
* Additional servers pull the image that the main server pushes. Without a registry image, each
|
||||
* server builds its own image, so the servers can run different code.
|
||||
*/
|
||||
private function ensureRegistryImageForMultipleServers(): void
|
||||
{
|
||||
if ($this->pull_request_id !== 0) {
|
||||
return;
|
||||
}
|
||||
if ($this->application->additional_servers()->doesntExist()) {
|
||||
return;
|
||||
}
|
||||
if (str($this->application->docker_registry_image_name)->isNotEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
throw new DeploymentException('Before deploying to multiple servers, you must set a Docker image in the General tab. More information: https://coolify.io/docs/knowledge-base/server/multiple-servers');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build-time names go into shell and Docker build commands, so they must be valid when the
|
||||
* deployment builds an image. Runtime-only variables only go into the .env file: existing ones
|
||||
@@ -2640,23 +2690,27 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
$this->prepare_builder_image(firstTry: false);
|
||||
}
|
||||
|
||||
private function deploy_to_additional_destinations()
|
||||
/**
|
||||
* @return int The number of deployments that Coolify queued on additional servers
|
||||
*/
|
||||
private function deploy_to_additional_destinations(): int
|
||||
{
|
||||
if ($this->application->additional_networks->count() === 0) {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
if ($this->pull_request_id !== 0) {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
$destination_ids = $this->application->additional_networks->pluck('id');
|
||||
if ($this->server->isSwarm()) {
|
||||
$this->application_deployment_queue->addLogEntry('Additional destinations are not supported in swarm mode.');
|
||||
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
if ($destination_ids->contains($this->destination->id)) {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
$queued = 0;
|
||||
foreach ($destination_ids as $destination_id) {
|
||||
$destination = StandaloneDocker::find($destination_id);
|
||||
if (! $destination) {
|
||||
@@ -2669,16 +2723,29 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
|
||||
continue;
|
||||
}
|
||||
$deployment_uuid = new_public_id();
|
||||
queue_application_deployment(
|
||||
// Deploy the exact commit and image tag of the main server, also for a rollback.
|
||||
$result = queue_application_deployment(
|
||||
deployment_uuid: $deployment_uuid,
|
||||
application: $this->application,
|
||||
commit: $this->commit,
|
||||
no_questions_asked: true,
|
||||
server: $server,
|
||||
destination: $destination,
|
||||
no_questions_asked: true,
|
||||
rollback: $this->rollback,
|
||||
docker_registry_image_tag: $this->application_deployment_queue->docker_registry_image_tag,
|
||||
parent_deployment_uuid: $this->deployment_uuid,
|
||||
);
|
||||
if ($result['status'] !== 'queued') {
|
||||
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name} was not queued: {$result['message']}", 'stderr');
|
||||
|
||||
continue;
|
||||
}
|
||||
$queued++;
|
||||
$deployment_url = base_url().'/project/'.data_get($this->application, 'environment.project.uuid').'/environment/'.data_get($this->application, 'environment.uuid').'/application/'.data_get($this->application, 'uuid')."/deployment/{$deployment_uuid}";
|
||||
$this->application_deployment_queue->addLogEntry("Deployment to {$server->name}. Logs: {$deployment_url}");
|
||||
}
|
||||
|
||||
return $queued;
|
||||
}
|
||||
|
||||
private function set_coolify_variables()
|
||||
@@ -5547,11 +5614,20 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
||||
|
||||
event(new ApplicationConfigurationChanged($this->application->team()->id));
|
||||
|
||||
if (! $this->only_this_server) {
|
||||
$this->deploy_to_additional_destinations();
|
||||
if (filled($this->application_deployment_queue->parent_deployment_uuid)) {
|
||||
$this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->sendDeploymentNotification(DeploymentSuccess::class);
|
||||
$queuedAdditionalDeployments = $this->only_this_server ? 0 : $this->deploy_to_additional_destinations();
|
||||
if ($queuedAdditionalDeployments === 0) {
|
||||
$this->sendDeploymentNotification(DeploymentSuccess::class);
|
||||
|
||||
return;
|
||||
}
|
||||
Cache::put("multi-server-deployment-queued:{$this->deployment_uuid}", true, now()->addDay());
|
||||
$this->sendMultiServerDeploymentNotification($this->deployment_uuid);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -5559,9 +5635,47 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
||||
*/
|
||||
private function handleFailedDeployment(): void
|
||||
{
|
||||
if (filled($this->application_deployment_queue->parent_deployment_uuid)) {
|
||||
$this->sendMultiServerDeploymentNotification($this->application_deployment_queue->parent_deployment_uuid);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->sendDeploymentNotification(DeploymentFailed::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* The main server queues one deployment for each additional server. When all are queued and the
|
||||
* last one ends, Coolify sends one notification: success if all servers succeeded, else failed
|
||||
* with a link to the first deployment that did not succeed.
|
||||
*/
|
||||
private function sendMultiServerDeploymentNotification(string $parentDeploymentUuid): void
|
||||
{
|
||||
if (! Cache::has("multi-server-deployment-queued:{$parentDeploymentUuid}")) {
|
||||
return;
|
||||
}
|
||||
$deployments = ApplicationDeploymentQueue::query()
|
||||
->where('parent_deployment_uuid', $parentDeploymentUuid)
|
||||
->get(['deployment_uuid', 'status']);
|
||||
|
||||
$endStatuses = [
|
||||
ApplicationDeploymentStatus::FINISHED->value,
|
||||
ApplicationDeploymentStatus::FAILED->value,
|
||||
ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
|
||||
];
|
||||
if ($deployments->contains(fn (ApplicationDeploymentQueue $deployment) => ! in_array($deployment->status, $endStatuses, true))) {
|
||||
return;
|
||||
}
|
||||
if (! Cache::add("multi-server-deployment-notified:{$parentDeploymentUuid}", true, now()->addDay())) {
|
||||
return;
|
||||
}
|
||||
|
||||
$unsuccessful = $deployments->first(fn (ApplicationDeploymentQueue $deployment) => $deployment->status !== ApplicationDeploymentStatus::FINISHED->value);
|
||||
$this->application->environment->project->team?->notify($unsuccessful
|
||||
? new DeploymentFailed($this->application, $unsuccessful->deployment_uuid, $this->preview)
|
||||
: new DeploymentSuccess($this->application, $parentDeploymentUuid, $this->preview));
|
||||
}
|
||||
|
||||
/**
|
||||
* Send deployment status notification to the team.
|
||||
*/
|
||||
|
||||
@@ -97,6 +97,10 @@ class ConnectApplication extends Component
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($reason = $application->persistentStorageUnavailableReason()) {
|
||||
throw new \Exception($reason);
|
||||
}
|
||||
|
||||
if ($application->persistentStorages()->where('standalone_sqlite_id', $this->database->id)->exists()) {
|
||||
throw new \Exception("{$application->name} already mounts this database volume.");
|
||||
}
|
||||
|
||||
@@ -240,6 +240,14 @@ class Storage extends Component
|
||||
return collect($this->fileStorage)->where('is_directory', true);
|
||||
}
|
||||
|
||||
/**
|
||||
* The reason why the resource cannot get a volume mount, or null if it can.
|
||||
*/
|
||||
public function getVolumeUnavailableReasonProperty(): ?string
|
||||
{
|
||||
return $this->resource instanceof Application ? $this->resource->persistentStorageUnavailableReason() : null;
|
||||
}
|
||||
|
||||
public function getVolumeCountProperty()
|
||||
{
|
||||
return $this->cachedVolumeCount;
|
||||
@@ -265,6 +273,12 @@ class Storage extends Component
|
||||
'mount_path' => 'required|string',
|
||||
], ValidationPatterns::volumeNameMessages());
|
||||
|
||||
if ($reason = $this->volumeUnavailableReason) {
|
||||
$this->dispatch('error', 'Failed to add volume.', $reason);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$name = $this->resource->uuid.'-'.$this->name;
|
||||
|
||||
LocalPersistentVolume::create([
|
||||
|
||||
@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Shared;
|
||||
use App\Actions\Application\StopApplicationOneServer;
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Events\ApplicationStatusChanged;
|
||||
use App\Models\Application;
|
||||
use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||
@@ -156,6 +157,14 @@ class Destination extends Component
|
||||
$server = Server::ownedByCurrentTeam()->findOrFail($server_id);
|
||||
$network = StandaloneDocker::ownedByCurrentTeam()->where('server_id', $server->id)->findOrFail($network_id);
|
||||
$this->authorize('update', $this->resource);
|
||||
$reason = $this->resource instanceof Application
|
||||
? $this->resource->additionalServersUnavailableReason()
|
||||
: 'Only applications can use multiple servers.';
|
||||
if ($reason) {
|
||||
$this->dispatch('error', 'Failed to add server.', $reason);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->resource->additional_networks()->syncWithoutDetaching([
|
||||
$network->id => ['server_id' => $server->id],
|
||||
|
||||
@@ -585,6 +585,33 @@ class Application extends BaseModel
|
||||
->withPivot('server_id', 'status');
|
||||
}
|
||||
|
||||
/**
|
||||
* The reason why this application cannot get a persistent volume, or null if it can.
|
||||
*/
|
||||
public function persistentStorageUnavailableReason(): ?string
|
||||
{
|
||||
if ($this->additional_servers()->exists()) {
|
||||
return 'Applications that use multiple servers cannot have persistent storage because volumes are not shared between servers.';
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The reason why this application cannot deploy to additional servers, or null if it can.
|
||||
*/
|
||||
public function additionalServersUnavailableReason(): ?string
|
||||
{
|
||||
if ($this->build_pack === 'dockercompose') {
|
||||
return 'Docker Compose applications cannot use multiple servers.';
|
||||
}
|
||||
if ($this->persistentStorages()->exists()) {
|
||||
return 'Applications with persistent storage cannot use multiple servers because volumes are not shared between servers.';
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public function is_public_repository(): bool
|
||||
{
|
||||
if (data_get($this, 'source.is_public')) {
|
||||
|
||||
@@ -39,6 +39,7 @@ use OpenApi\Attributes as OA;
|
||||
'deployment_url' => ['type' => 'string'],
|
||||
'destination_id' => ['type' => 'string'],
|
||||
'only_this_server' => ['type' => 'boolean'],
|
||||
'parent_deployment_uuid' => ['type' => 'string', 'nullable' => true],
|
||||
'rollback' => ['type' => 'boolean'],
|
||||
'commit_message' => ['type' => 'string'],
|
||||
],
|
||||
@@ -105,6 +106,7 @@ class ApplicationDeploymentQueue extends Model
|
||||
'deployment_url',
|
||||
'destination_id',
|
||||
'only_this_server',
|
||||
'parent_deployment_uuid',
|
||||
'rollback',
|
||||
'commit_message',
|
||||
'is_api',
|
||||
|
||||
@@ -4,10 +4,12 @@ namespace App\Models;
|
||||
|
||||
use App\Events\FileStorageChanged;
|
||||
use App\Jobs\ServerStorageSaveJob;
|
||||
use Closure;
|
||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Relations\MorphMany;
|
||||
use Illuminate\Database\Eloquent\Relations\MorphTo;
|
||||
use Illuminate\Support\Collection;
|
||||
use Symfony\Component\Yaml\Yaml;
|
||||
|
||||
class LocalFileVolume extends BaseModel
|
||||
@@ -236,21 +238,21 @@ class LocalFileVolume extends BaseModel
|
||||
return (string) $content;
|
||||
}
|
||||
|
||||
public function deleteStorageOnServer()
|
||||
/**
|
||||
* Without a server, Coolify deletes the file on every server of the resource.
|
||||
*/
|
||||
public function deleteStorageOnServer(?Server $server = null)
|
||||
{
|
||||
if ($this->is_host_file) {
|
||||
return;
|
||||
}
|
||||
if (is_null($server)) {
|
||||
return $this->runOnEveryServer(fn (Server $server) => $this->deleteStorageOnServer($server));
|
||||
}
|
||||
|
||||
$this->load(['service']);
|
||||
$isService = data_get($this->resource, 'service');
|
||||
if ($isService) {
|
||||
$workdir = $this->resource->service->workdir();
|
||||
$server = $this->resource->service->server;
|
||||
} else {
|
||||
$workdir = $this->resource->workdir();
|
||||
$server = $this->resource->destination->server;
|
||||
}
|
||||
$workdir = $isService ? $this->resource->service->workdir() : $this->resource->workdir();
|
||||
$commands = collect([]);
|
||||
$path = data_get_str($this, 'fs_path');
|
||||
if ($path->startsWith('.')) {
|
||||
@@ -282,21 +284,21 @@ class LocalFileVolume extends BaseModel
|
||||
}
|
||||
}
|
||||
|
||||
public function saveStorageOnServer()
|
||||
/**
|
||||
* Without a server, Coolify writes the file on every server of the resource.
|
||||
*/
|
||||
public function saveStorageOnServer(?Server $server = null)
|
||||
{
|
||||
if ($this->is_host_file) {
|
||||
return;
|
||||
}
|
||||
if (is_null($server)) {
|
||||
return $this->runOnEveryServer(fn (Server $server) => $this->saveStorageOnServer($server));
|
||||
}
|
||||
|
||||
$this->load(['service']);
|
||||
$isService = data_get($this->resource, 'service');
|
||||
if ($isService) {
|
||||
$workdir = $this->resource->service->workdir();
|
||||
$server = $this->resource->service->server;
|
||||
} else {
|
||||
$workdir = $this->resource->workdir();
|
||||
$server = $this->resource->destination->server;
|
||||
}
|
||||
$workdir = $isService ? $this->resource->service->workdir() : $this->resource->workdir();
|
||||
$commands = collect([]);
|
||||
$escapedWorkdir = escapeshellarg($workdir);
|
||||
|
||||
@@ -555,6 +557,49 @@ class LocalFileVolume extends BaseModel
|
||||
: $owner->workdir();
|
||||
}
|
||||
|
||||
/**
|
||||
* The main server of the resource, and the additional servers of an application.
|
||||
*
|
||||
* @return Collection<int, Server>
|
||||
*/
|
||||
public function servers(): Collection
|
||||
{
|
||||
$this->load(['service']);
|
||||
if (data_get($this->resource, 'service')) {
|
||||
return collect([$this->resource->service->server]);
|
||||
}
|
||||
|
||||
$servers = collect([$this->resource->destination->server]);
|
||||
if ($this->resource instanceof Application) {
|
||||
$servers = $servers->merge($this->resource->additional_servers);
|
||||
}
|
||||
|
||||
return $servers->filter()->unique('id')->values();
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the callback on every server, also when one server fails, and then throws the first error.
|
||||
*
|
||||
* @param Closure(Server): mixed $callback
|
||||
*/
|
||||
protected function runOnEveryServer(Closure $callback): mixed
|
||||
{
|
||||
$result = null;
|
||||
$firstError = null;
|
||||
foreach ($this->servers() as $server) {
|
||||
try {
|
||||
$result = $callback($server);
|
||||
} catch (\Throwable $e) {
|
||||
$firstError ??= $e;
|
||||
}
|
||||
}
|
||||
if ($firstError) {
|
||||
throw $firstError;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Application, Service or standalone database that owns the storage directory.
|
||||
*/
|
||||
|
||||
@@ -12,7 +12,7 @@ use App\Models\StandaloneDocker;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Spatie\Url\Url;
|
||||
|
||||
function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null)
|
||||
function queue_application_deployment(Application $application, string $deployment_uuid, ?int $pull_request_id = 0, ?string $commit = null, bool $force_rebuild = false, bool $is_webhook = false, bool $is_api = false, bool $restart_only = false, ?string $git_type = null, bool $no_questions_asked = false, ?Server $server = null, ?StandaloneDocker $destination = null, bool $only_this_server = false, bool $rollback = false, ?string $docker_registry_image_tag = null, ?string $parent_deployment_uuid = null)
|
||||
{
|
||||
$commit = $commit ?: ($application->git_commit_sha ?: 'HEAD');
|
||||
$commit = validateGitRef($commit, 'deployment commit');
|
||||
@@ -31,7 +31,7 @@ function queue_application_deployment(Application $application, string $deployme
|
||||
$destination_id = $destination->id;
|
||||
}
|
||||
|
||||
$admission = DB::transaction(function () use ($application, $application_id, $commit, $deployment_uuid, $deployment_url, $destination_id, $docker_registry_image_tag, $force_rebuild, $git_type, $is_api, $is_webhook, $no_questions_asked, $only_this_server, $pull_request_id, $restart_only, $rollback, $server_id, $server_name) {
|
||||
$admission = DB::transaction(function () use ($application, $application_id, $commit, $deployment_uuid, $deployment_url, $destination_id, $docker_registry_image_tag, $force_rebuild, $git_type, $is_api, $is_webhook, $no_questions_asked, $only_this_server, $parent_deployment_uuid, $pull_request_id, $restart_only, $rollback, $server_id, $server_name) {
|
||||
// Lock stable rows because an empty deployment queue has no row to lock.
|
||||
Application::query()->whereKey($application_id)->lockForUpdate()->firstOrFail();
|
||||
$serverForQueueCheck = Server::query()->whereKey($server_id)->lockForUpdate()->firstOrFail();
|
||||
@@ -81,6 +81,7 @@ function queue_application_deployment(Application $application, string $deployme
|
||||
'rollback' => $rollback,
|
||||
'git_type' => $git_type,
|
||||
'only_this_server' => $only_this_server,
|
||||
'parent_deployment_uuid' => $parent_deployment_uuid,
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Links the deployments on additional servers to the main deployment, so Coolify can send one
|
||||
* notification after all servers finish.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('application_deployment_queues', function (Blueprint $table) {
|
||||
$table->string('parent_deployment_uuid')->nullable()->index();
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('application_deployment_queues', function (Blueprint $table) {
|
||||
$table->dropIndex(['parent_deployment_uuid']);
|
||||
$table->dropColumn('parent_deployment_uuid');
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -23241,6 +23241,10 @@
|
||||
"only_this_server": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"parent_deployment_uuid": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"rollback": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
||||
@@ -14873,6 +14873,9 @@ components:
|
||||
type: string
|
||||
only_this_server:
|
||||
type: boolean
|
||||
parent_deployment_uuid:
|
||||
type: string
|
||||
nullable: true
|
||||
rollback:
|
||||
type: boolean
|
||||
commit_message:
|
||||
|
||||
@@ -112,6 +112,11 @@
|
||||
})
|
||||
}
|
||||
})">
|
||||
@if ($this->volumeUnavailableReason)
|
||||
<x-callout type="warning" title="Volume mounts are unavailable" class="w-full">
|
||||
{{ $this->volumeUnavailableReason }}
|
||||
</x-callout>
|
||||
@else
|
||||
<form class="flex w-full flex-col gap-4"
|
||||
wire:submit='submitPersistentVolume'>
|
||||
<p class="text-[13px] leading-5 text-neutral-500 dark:text-fg-dim">
|
||||
@@ -130,6 +135,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
@endif
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -482,6 +482,70 @@ describe('Application multi-destination cross-team', function () {
|
||||
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
|
||||
});
|
||||
|
||||
test('attaches a destination on another server of the same team', function () {
|
||||
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
|
||||
|
||||
$this->withHeaders($this->headers)
|
||||
->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
|
||||
'destination_uuid' => $secondDestination->uuid,
|
||||
])
|
||||
->assertSuccessful();
|
||||
|
||||
expect($this->application->fresh()->additional_networks)->toHaveCount(1);
|
||||
});
|
||||
|
||||
test('rejects an additional server for an application with persistent storage', function () {
|
||||
LocalPersistentVolume::create([
|
||||
'name' => 'app-data-'.$this->application->uuid,
|
||||
'mount_path' => '/data',
|
||||
'resource_id' => $this->application->id,
|
||||
'resource_type' => $this->application->getMorphClass(),
|
||||
]);
|
||||
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
|
||||
|
||||
$this->withHeaders($this->headers)
|
||||
->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
|
||||
'destination_uuid' => $secondDestination->uuid,
|
||||
])
|
||||
->assertUnprocessable()
|
||||
->assertJsonPath('message', 'Applications with persistent storage cannot use multiple servers because volumes are not shared between servers.');
|
||||
|
||||
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
|
||||
});
|
||||
|
||||
test('rejects an additional server for a Docker Compose application', function () {
|
||||
$this->application->update(['build_pack' => 'dockercompose']);
|
||||
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
|
||||
|
||||
$this->withHeaders($this->headers)
|
||||
->postJson("/api/v1/applications/{$this->application->uuid}/destinations", [
|
||||
'destination_uuid' => $secondDestination->uuid,
|
||||
])
|
||||
->assertUnprocessable();
|
||||
|
||||
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
|
||||
});
|
||||
|
||||
test('rejects a volume for an application with an additional server', function () {
|
||||
$secondServer = Server::factory()->create(['team_id' => $this->team->id]);
|
||||
$secondDestination = StandaloneDocker::where('server_id', $secondServer->id)->firstOrFail();
|
||||
$this->application->additional_networks()->attach($secondDestination->id, ['server_id' => $secondServer->id]);
|
||||
|
||||
$this->withHeaders($this->headers)
|
||||
->postJson("/api/v1/applications/{$this->application->uuid}/storages", [
|
||||
'type' => 'persistent',
|
||||
'name' => 'data',
|
||||
'mount_path' => '/data',
|
||||
])
|
||||
->assertUnprocessable()
|
||||
->assertJsonPath('message', 'Applications that use multiple servers cannot have persistent storage because volumes are not shared between servers.');
|
||||
|
||||
expect($this->application->persistentStorages()->count())->toBe(0);
|
||||
});
|
||||
|
||||
test('lists primary destination', function () {
|
||||
$response = $this->withHeaders($this->headers)
|
||||
->getJson("/api/v1/applications/{$this->application->uuid}/destinations");
|
||||
|
||||
@@ -200,8 +200,8 @@ function runComposeStart(object $test, array $properties = []): ContentFilesUpDe
|
||||
foreach ([
|
||||
'application' => $test->application->fresh(),
|
||||
'application_deployment_queue' => $queue,
|
||||
'server' => $test->server,
|
||||
'mainServer' => $test->server,
|
||||
'server' => $test->server->fresh(),
|
||||
'mainServer' => $test->server->fresh(),
|
||||
'deployment_uuid' => 'deployment-uuid',
|
||||
'workdir' => '/artifacts/deployment-uuid',
|
||||
'configuration_dir' => $test->application->workdir(),
|
||||
@@ -343,18 +343,22 @@ test('a preserve-repository deployment writes content files before docker compos
|
||||
->and($write)->toBeLessThan(ContentFilesUpRecorder::indexOf('deployment', ' up -d'));
|
||||
});
|
||||
|
||||
test('a deployment to another server does not check the content files of the main server', function () {
|
||||
test('a deployment to an additional server writes missing content files on that server', function () {
|
||||
$volume = contentFilesVolume($this->application, 'config/app.conf');
|
||||
$otherServer = Server::factory()->create([
|
||||
'team_id' => $this->team->id,
|
||||
'private_key_id' => $this->server->private_key_id,
|
||||
]);
|
||||
])->fresh();
|
||||
fakeContentFilesServer([$volume->fs_path => 'missing']);
|
||||
|
||||
runComposeStart($this, ['server' => $otherServer, 'mainServer' => $otherServer]);
|
||||
|
||||
expect(contentFilesStateChecks())->toBe([]);
|
||||
contentFilesAssertNoWrite();
|
||||
$write = ContentFilesUpRecorder::indexOf('ssh', 'base64 -d | tee '.escapeshellarg($volume->fs_path));
|
||||
expect(contentFilesStateChecks())->toHaveCount(1)
|
||||
->and(contentFilesStateChecks()[0])->toContain("@'{$otherServer->ip}'")
|
||||
->and($write)->not->toBeNull()
|
||||
->and(ContentFilesUpRecorder::ssh()[$write])->toContain("@'{$otherServer->ip}'")
|
||||
->and(ContentFilesUpRecorder::ssh()[$write])->not->toContain("@'{$this->server->ip}'");
|
||||
});
|
||||
|
||||
test('the content file check and write are safe for non-root servers', function () {
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
<?php
|
||||
|
||||
use App\Actions\Application\StopApplication;
|
||||
use App\Enums\ApplicationDeploymentStatus;
|
||||
use App\Exceptions\DeploymentException;
|
||||
use App\Jobs\ApplicationDeploymentJob;
|
||||
use App\Livewire\Project\Database\Sqlite\ConnectApplication;
|
||||
use App\Livewire\Project\Service\Storage;
|
||||
use App\Livewire\Project\Shared\Destination;
|
||||
use App\Models\Application;
|
||||
use App\Models\ApplicationDeploymentQueue;
|
||||
use App\Models\Environment;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\LocalFileVolume;
|
||||
use App\Models\LocalPersistentVolume;
|
||||
use App\Models\Project;
|
||||
use App\Models\Server;
|
||||
use App\Models\StandaloneDocker;
|
||||
use App\Models\StandaloneSqlite;
|
||||
use App\Models\Team;
|
||||
use App\Models\User;
|
||||
use App\Notifications\Application\DeploymentFailed;
|
||||
use App\Notifications\Application\DeploymentSuccess;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Livewire\Livewire;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Server::flushIdentityMap();
|
||||
Queue::fake();
|
||||
Notification::fake();
|
||||
config()->set('cache.default', 'array');
|
||||
|
||||
InstanceSettings::unguarded(fn () => InstanceSettings::query()->create(['id' => 0]));
|
||||
|
||||
$this->team = Team::factory()->create();
|
||||
$this->team->emailNotificationSettings->update([
|
||||
'smtp_enabled' => true,
|
||||
'deployment_success_email_notifications' => true,
|
||||
'deployment_failure_email_notifications' => true,
|
||||
]);
|
||||
$this->user = User::factory()->create();
|
||||
$this->user->teams()->attach($this->team, ['role' => 'owner']);
|
||||
|
||||
$this->mainServer = Server::factory()->create(['team_id' => $this->team->id, 'name' => 'main-server']);
|
||||
$this->mainDestination = StandaloneDocker::where('server_id', $this->mainServer->id)->firstOrFail();
|
||||
$this->extraServer = Server::factory()->create(['team_id' => $this->team->id, 'name' => 'extra-server']);
|
||||
$this->extraDestination = StandaloneDocker::where('server_id', $this->extraServer->id)->firstOrFail();
|
||||
|
||||
$project = Project::factory()->create(['team_id' => $this->team->id]);
|
||||
$environment = $project->environments()->first() ?? Environment::factory()->create(['project_id' => $project->id]);
|
||||
|
||||
$this->application = Application::factory()->create([
|
||||
'environment_id' => $environment->id,
|
||||
'destination_id' => $this->mainDestination->id,
|
||||
'destination_type' => $this->mainDestination->getMorphClass(),
|
||||
'build_pack' => 'dockerfile',
|
||||
]);
|
||||
$this->application->additional_networks()->attach($this->extraDestination->id, ['server_id' => $this->extraServer->id]);
|
||||
$this->application->refresh();
|
||||
});
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $properties
|
||||
*/
|
||||
function makeMultiServerDeploymentJob(array $properties): array
|
||||
{
|
||||
$reflection = new ReflectionClass(ApplicationDeploymentJob::class);
|
||||
$job = $reflection->newInstanceWithoutConstructor();
|
||||
foreach (['pull_request_id' => 0, 'preview' => null, 'only_this_server' => false, 'commit' => 'HEAD', 'rollback' => false, ...$properties] as $name => $value) {
|
||||
$reflection->getProperty($name)->setValue($job, $value);
|
||||
}
|
||||
|
||||
return [$job, $reflection];
|
||||
}
|
||||
|
||||
function callMultiServerDeploymentJob(array $jobAndReflection, string $method, mixed ...$arguments): mixed
|
||||
{
|
||||
[$job, $reflection] = $jobAndReflection;
|
||||
|
||||
return $reflection->getMethod($method)->invoke($job, ...$arguments);
|
||||
}
|
||||
|
||||
function createMultiServerDeployment(Application $application, Server $server, string $status, ?string $parentDeploymentUuid = null): ApplicationDeploymentQueue
|
||||
{
|
||||
return ApplicationDeploymentQueue::create([
|
||||
'application_id' => $application->id,
|
||||
'deployment_uuid' => new_public_id(),
|
||||
'server_id' => $server->id,
|
||||
'status' => $status,
|
||||
'parent_deployment_uuid' => $parentDeploymentUuid,
|
||||
]);
|
||||
}
|
||||
|
||||
describe('registry image guard', function () {
|
||||
test('fails a multi-server deployment without a registry image', function () {
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application]);
|
||||
|
||||
expect(fn () => callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers'))
|
||||
->toThrow(DeploymentException::class, 'Before deploying to multiple servers');
|
||||
});
|
||||
|
||||
test('allows a multi-server deployment with a registry image', function () {
|
||||
$this->application->update(['docker_registry_image_name' => 'registry.example.com/app']);
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application->fresh()]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
|
||||
})->throwsNoExceptions();
|
||||
|
||||
test('allows preview deployments, which run only on the main server', function () {
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'pull_request_id' => 7]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'ensureRegistryImageForMultipleServers');
|
||||
})->throwsNoExceptions();
|
||||
});
|
||||
|
||||
function makeVolumeWarningJob(Application $application, array &$logEntries): array
|
||||
{
|
||||
$queue = Mockery::mock(ApplicationDeploymentQueue::class);
|
||||
$queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message, string $type = 'stdout') use (&$logEntries) {
|
||||
$logEntries[] = [$message, $type];
|
||||
});
|
||||
|
||||
return makeMultiServerDeploymentJob(['application' => $application, 'application_deployment_queue' => $queue]);
|
||||
}
|
||||
|
||||
describe('volume warning', function () {
|
||||
test('warns in the deployment log when an application with a volume uses multiple servers', function () {
|
||||
LocalPersistentVolume::create([
|
||||
'name' => 'app-data-'.$this->application->uuid,
|
||||
'mount_path' => '/data',
|
||||
'resource_id' => $this->application->id,
|
||||
'resource_type' => $this->application->getMorphClass(),
|
||||
]);
|
||||
$logEntries = [];
|
||||
$job = makeVolumeWarningJob($this->application->fresh(), $logEntries);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'warnAboutVolumesOnMultipleServers');
|
||||
|
||||
expect($logEntries)->toBe([[
|
||||
'Warning: This application uses multiple servers and has persistent storage. Volumes are not shared between servers, so each server has its own data.',
|
||||
'stderr',
|
||||
]]);
|
||||
});
|
||||
|
||||
test('does not warn when the application has no volume', function () {
|
||||
$logEntries = [];
|
||||
$job = makeVolumeWarningJob($this->application, $logEntries);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'warnAboutVolumesOnMultipleServers');
|
||||
|
||||
expect($logEntries)->toBe([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined deployment notification', function () {
|
||||
test('links queued additional deployments to the main deployment', function () {
|
||||
$parent = createMultiServerDeployment($this->application, $this->mainServer, ApplicationDeploymentStatus::FINISHED->value);
|
||||
$job = makeMultiServerDeploymentJob([
|
||||
'application' => $this->application,
|
||||
'application_deployment_queue' => $parent,
|
||||
'deployment_uuid' => $parent->deployment_uuid,
|
||||
'server' => $this->mainServer,
|
||||
'mainServer' => $this->mainServer,
|
||||
'destination' => $this->mainDestination,
|
||||
]);
|
||||
|
||||
expect(callMultiServerDeploymentJob($job, 'deploy_to_additional_destinations'))->toBe(1);
|
||||
|
||||
$child = ApplicationDeploymentQueue::where('server_id', $this->extraServer->id)->sole();
|
||||
expect($child->parent_deployment_uuid)->toBe($parent->deployment_uuid);
|
||||
});
|
||||
|
||||
test('additional servers deploy the same commit and image tag as a rollback on the main server', function () {
|
||||
$parent = ApplicationDeploymentQueue::create([
|
||||
'application_id' => $this->application->id,
|
||||
'deployment_uuid' => new_public_id(),
|
||||
'server_id' => $this->mainServer->id,
|
||||
'status' => ApplicationDeploymentStatus::FINISHED->value,
|
||||
'commit' => 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678',
|
||||
'rollback' => true,
|
||||
'docker_registry_image_tag' => 'v1.2.3',
|
||||
]);
|
||||
$job = makeMultiServerDeploymentJob([
|
||||
'application' => $this->application,
|
||||
'application_deployment_queue' => $parent,
|
||||
'deployment_uuid' => $parent->deployment_uuid,
|
||||
'server' => $this->mainServer,
|
||||
'mainServer' => $this->mainServer,
|
||||
'destination' => $this->mainDestination,
|
||||
'commit' => $parent->commit,
|
||||
'rollback' => true,
|
||||
]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'deploy_to_additional_destinations');
|
||||
|
||||
$child = ApplicationDeploymentQueue::where('server_id', $this->extraServer->id)->sole();
|
||||
expect($child->commit)->toBe('a1b2c3d4e5f60718293a4b5c6d7e8f9012345678')
|
||||
->and((bool) $child->rollback)->toBeTrue()
|
||||
->and($child->docker_registry_image_tag)->toBe('v1.2.3');
|
||||
});
|
||||
|
||||
test('does not notify while an additional server is still deploying', function () {
|
||||
$parentUuid = new_public_id();
|
||||
Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
|
||||
$failed = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FAILED->value, $parentUuid);
|
||||
createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::IN_PROGRESS->value, $parentUuid);
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $failed]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'handleFailedDeployment');
|
||||
|
||||
Notification::assertNothingSent();
|
||||
});
|
||||
|
||||
test('does not notify before the main deployment queued all additional servers', function () {
|
||||
$parentUuid = new_public_id();
|
||||
$finished = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $finished]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
|
||||
|
||||
Notification::assertNothingSent();
|
||||
});
|
||||
|
||||
test('sends one success notification after all additional servers succeed', function () {
|
||||
$parentUuid = new_public_id();
|
||||
Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
|
||||
$first = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
|
||||
createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $first]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
|
||||
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
|
||||
|
||||
Notification::assertSentToTimes($this->team, DeploymentSuccess::class, 1);
|
||||
Notification::assertSentTo($this->team, DeploymentSuccess::class, fn (DeploymentSuccess $notification) => $notification->deployment_uuid === $parentUuid);
|
||||
Notification::assertNotSentTo($this->team, DeploymentFailed::class);
|
||||
});
|
||||
|
||||
test('sends a failed notification that links to the failed server deployment', function () {
|
||||
$parentUuid = new_public_id();
|
||||
Cache::put("multi-server-deployment-queued:{$parentUuid}", true);
|
||||
$finished = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FINISHED->value, $parentUuid);
|
||||
$failed = createMultiServerDeployment($this->application, $this->extraServer, ApplicationDeploymentStatus::FAILED->value, $parentUuid);
|
||||
$job = makeMultiServerDeploymentJob(['application' => $this->application, 'application_deployment_queue' => $finished]);
|
||||
|
||||
callMultiServerDeploymentJob($job, 'sendMultiServerDeploymentNotification', $parentUuid);
|
||||
|
||||
Notification::assertSentTo($this->team, DeploymentFailed::class, fn (DeploymentFailed $notification) => $notification->deployment_uuid === $failed->deployment_uuid);
|
||||
Notification::assertNotSentTo($this->team, DeploymentSuccess::class);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Records the deployment commands instead of running them on a server.
|
||||
*/
|
||||
class MultiServerRecordingDeploymentJob extends ApplicationDeploymentJob
|
||||
{
|
||||
/** @var list<string> */
|
||||
public array $recordedCommands = [];
|
||||
|
||||
public function __construct() {}
|
||||
|
||||
public function execute_remote_command(...$commands): void
|
||||
{
|
||||
foreach ($commands as $command) {
|
||||
$this->recordedCommands[] = $command['command'] ?? $command[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function makeInlineDockerfileJob(Application $application, bool $isAdditionalServer): array
|
||||
{
|
||||
$reflection = new ReflectionClass(ApplicationDeploymentJob::class);
|
||||
$job = new MultiServerRecordingDeploymentJob;
|
||||
$queue = Mockery::mock(ApplicationDeploymentQueue::class);
|
||||
$queue->shouldReceive('addLogEntry');
|
||||
foreach ([
|
||||
'application' => $application,
|
||||
'application_deployment_queue' => $queue,
|
||||
'is_this_additional_server' => $isAdditionalServer,
|
||||
'pull_request_id' => 0,
|
||||
] as $name => $value) {
|
||||
$reflection->getProperty($name)->setValue($job, $value);
|
||||
}
|
||||
$reflection->getMethod('generate_image_names')->invoke($job);
|
||||
|
||||
return [$job, $reflection];
|
||||
}
|
||||
|
||||
describe('inline Dockerfile image on additional servers', function () {
|
||||
beforeEach(function () {
|
||||
$this->application->update([
|
||||
'dockerfile' => "FROM nginx:alpine\n",
|
||||
'docker_registry_image_name' => 'registry.example.com/nginx-multi',
|
||||
]);
|
||||
});
|
||||
|
||||
test('an additional server pulls the image that the main server pushed', function () {
|
||||
[$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), true);
|
||||
|
||||
expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeTrue()
|
||||
->and($job->recordedCommands)->toBe(["docker pull 'registry.example.com/nginx-multi:latest'"]);
|
||||
});
|
||||
|
||||
test('the main server builds the image', function () {
|
||||
[$job, $reflection] = makeInlineDockerfileJob($this->application->fresh(), false);
|
||||
|
||||
expect($reflection->getMethod('pull_image_for_additional_server')->invoke($job))->toBeFalse()
|
||||
->and($job->recordedCommands)->toBe([]);
|
||||
});
|
||||
});
|
||||
|
||||
test('stopping continues on the other servers when one server is not functional', function () {
|
||||
$this->mainServer->settings->update(['is_reachable' => false]);
|
||||
$this->extraServer->settings->update(['is_reachable' => false]);
|
||||
|
||||
$result = StopApplication::run($this->application->fresh());
|
||||
|
||||
expect($result)->toContain('main-server')->toContain('extra-server');
|
||||
});
|
||||
|
||||
test('file storages of an application belong to all of its servers', function () {
|
||||
$file = LocalFileVolume::create([
|
||||
'fs_path' => '/data/coolify/applications/'.$this->application->uuid.'/config.yml',
|
||||
'mount_path' => '/app/config.yml',
|
||||
'content' => 'key: value',
|
||||
'resource_id' => $this->application->id,
|
||||
'resource_type' => $this->application->getMorphClass(),
|
||||
]);
|
||||
|
||||
expect($file->servers()->pluck('id')->all())->toBe([$this->mainServer->id, $this->extraServer->id]);
|
||||
});
|
||||
|
||||
describe('adding a server in the UI', function () {
|
||||
beforeEach(function () {
|
||||
$this->application->additional_networks()->detach();
|
||||
$this->actingAs($this->user);
|
||||
session(['currentTeam' => $this->team]);
|
||||
});
|
||||
|
||||
test('rejects an application with persistent storage', function () {
|
||||
LocalPersistentVolume::create([
|
||||
'name' => 'app-data-'.$this->application->uuid,
|
||||
'mount_path' => '/data',
|
||||
'resource_id' => $this->application->id,
|
||||
'resource_type' => $this->application->getMorphClass(),
|
||||
]);
|
||||
|
||||
Livewire::test(Destination::class, ['resource' => $this->application->fresh()])
|
||||
->call('addServer', $this->extraDestination->id, $this->extraServer->id)
|
||||
->assertDispatched('error');
|
||||
|
||||
expect($this->application->fresh()->additional_networks)->toHaveCount(0);
|
||||
});
|
||||
|
||||
test('adds a server to an application without persistent storage', function () {
|
||||
Livewire::test(Destination::class, ['resource' => $this->application->fresh()])
|
||||
->call('addServer', $this->extraDestination->id, $this->extraServer->id);
|
||||
|
||||
expect($this->application->fresh()->additional_networks)->toHaveCount(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('adding a volume to an application with additional servers', function () {
|
||||
beforeEach(function () {
|
||||
$this->actingAs($this->user);
|
||||
session(['currentTeam' => $this->team]);
|
||||
});
|
||||
|
||||
test('the storage page rejects the volume', function () {
|
||||
Livewire::test(Storage::class, ['resource' => $this->application->fresh()])
|
||||
->assertSee('Volume mounts are unavailable')
|
||||
->set('name', 'data')
|
||||
->set('mount_path', '/data')
|
||||
->call('submitPersistentVolume')
|
||||
->assertDispatched('error');
|
||||
|
||||
expect($this->application->persistentStorages()->count())->toBe(0);
|
||||
});
|
||||
|
||||
test('connecting a SQLite database rejects the volume', function () {
|
||||
$sqlite = StandaloneSqlite::create([
|
||||
'name' => 'app-sqlite',
|
||||
'environment_id' => $this->application->environment_id,
|
||||
'destination_id' => $this->mainDestination->id,
|
||||
'destination_type' => $this->mainDestination->getMorphClass(),
|
||||
]);
|
||||
|
||||
Livewire::test(ConnectApplication::class, ['database' => $sqlite])
|
||||
->set('applicationUuid', $this->application->uuid)
|
||||
->set('mountPath', '/app/database')
|
||||
->call('connect')
|
||||
->assertNoRedirect();
|
||||
|
||||
expect($this->application->persistentStorages()->count())->toBe(0);
|
||||
});
|
||||
|
||||
test('an application on one server can still add a volume', function () {
|
||||
$this->application->additional_networks()->detach();
|
||||
|
||||
Livewire::test(Storage::class, ['resource' => $this->application->fresh()])
|
||||
->set('name', 'data')
|
||||
->set('mount_path', '/data')
|
||||
->call('submitPersistentVolume')
|
||||
->assertNotDispatched('error')
|
||||
->assertDontSee('Volume mounts are unavailable');
|
||||
|
||||
expect($this->application->persistentStorages()->count())->toBe(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user