withoutVite(); Server::flushIdentityMap(); InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); config(['constants.ssh.mux_enabled' => false]); Notification::fake(); Process::fake(['*' => Process::result(output: '')]); $this->user = User::factory()->create(); $this->team = Team::factory()->create(); $this->user->teams()->attach($this->team, ['role' => 'owner']); $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); $server = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $privateKey->id]); $server->settings()->update(['is_reachable' => true, 'is_usable' => true]); $this->destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); $this->project = Project::factory()->create(['team_id' => $this->team->id]); $this->environment = $this->project->environments()->first() ?? Environment::factory()->create(['project_id' => $this->project->id]); $this->service = Service::factory()->create([ 'docker_compose_raw' => CLONE_SAFETY_SERVICE_COMPOSE, 'environment_id' => $this->environment->id, 'server_id' => $server->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(), ]); $this->service->parse(); $this->actingAs($this->user); session(['currentTeam' => $this->team]); }); /** * Runs the queued jobs with the database driver, so every chain step and the catch callback * are serialized and restored like on a Horizon worker. */ function cloneSafetyUseDatabaseQueue(): void { Schema::create('jobs', function (Blueprint $table) { $table->id(); $table->string('queue')->index(); $table->longText('payload'); $table->unsignedTinyInteger('attempts'); $table->unsignedInteger('reserved_at')->nullable(); $table->unsignedInteger('available_at'); $table->unsignedInteger('created_at'); }); config(['queue.default' => 'database']); } /** * The worker runs in the test process, so a high `--memory` keeps the default 128 MB limit from * stopping it after the first chain step when earlier tests have grown the process. */ function cloneSafetyWorkQueue(): void { Artisan::call('queue:work', ['connection' => 'database', '--stop-when-empty' => true, '--queue' => 'high', '--sleep' => 0, '--memory' => 4096]); } /** The volume copy (`cp -a` in a helper container) fails like on a full disk; every other remote command works. */ function cloneSafetyFailVolumeCopy(): void { Process::fake(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), 'cp -a') ? Process::result(errorOutput: 'No space left on device', exitCode: 1) : Process::result(output: '')); } /** `docker ps` (running containers only) lists the given container names; every other remote command works. */ function cloneSafetyListRunningContainers(string ...$names): void { Process::fake(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), 'docker ps --filter') ? Process::result(output: implode("\n", $names)) : Process::result(output: '')); } function cloneSafetyMarkServiceRunning(Service $service): void { $service->applications()->update(['status' => 'running:healthy']); $service->databases()->update(['status' => 'running:healthy']); } test('a failed volume copy starts the running source service again and notifies the team', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); cloneSafetyFailVolumeCopy(); cloneSafetyMarkServiceRunning($this->service); StopService::shouldRun()->once()->withArgs(fn (Service $service) => $service->is($this->service)); StartService::shouldRun()->once()->withArgs(fn (Service $service) => $service->is($this->service)); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); expect(DB::table('failed_jobs')->count())->toBe(1); Notification::assertSentTo($this->team, GeneralNotification::class, function (GeneralNotification $notification) { return $notification->success === false && str_contains($notification->message, $this->service->name) && str_contains($notification->message, 'No space left on device'); }); }); test('a volume copy of a stopped source service neither stops nor starts it', function () { cloneSafetyUseDatabaseQueue(); StopService::shouldRun()->never(); StartService::shouldRun()->never(); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); expect(DB::table('failed_jobs')->count())->toBe(0); Notification::assertNothingSent(); }); test('a successful volume copy stops the running source service and starts it again', function () { cloneSafetyUseDatabaseQueue(); cloneSafetyMarkServiceRunning($this->service); StopService::shouldRun()->once(); StartService::shouldRun()->once(); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); expect(DB::table('failed_jobs')->count())->toBe(0); Notification::assertNothingSent(); }); test('a failed stop that leaves some containers of the source service running starts only the stopped containers again', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); cloneSafetyMarkServiceRunning($this->service); cloneSafetyListRunningContainers('web-'.$this->service->uuid); StopService::shouldRun()->once()->andReturn('docker stop timed out'); StartService::shouldRun()->never(); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); $workdir = $this->service->workdir(); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), "docker compose --project-directory {$workdir} -f {$workdir}/docker-compose.yml --project-name {$this->service->uuid} up -d --no-recreate < /dev/null")); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), '--force-recreate')); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => $notification->success === false && str_contains($notification->message, 'did not copy') && str_contains($notification->message, 'started its stopped containers again') && str_contains($notification->message, 'docker stop timed out')); }); test('a failed start of the stopped containers of a partly stopped source service is in the notification', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); cloneSafetyMarkServiceRunning($this->service); Process::fake(fn (PendingProcess $process) => match (true) { str_contains(remoteProcessScript($process), 'docker ps --filter') => Process::result(output: 'web-'.$this->service->uuid), str_contains(remoteProcessScript($process), 'up -d --no-recreate') => Process::result(errorOutput: 'Cannot connect to the Docker daemon', exitCode: 1), default => Process::result(output: ''), }); StopService::shouldRun()->once()->andReturn(null); StartService::shouldRun()->never(); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy') && str_contains($notification->message, 'could not start its stopped containers again')); }); test('a stop that leaves containers of the source service running copies no data', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); cloneSafetyMarkServiceRunning($this->service); cloneSafetyListRunningContainers('web-'.$this->service->uuid); StopService::shouldRun()->once()->andReturn(null); StartService::shouldRun()->never(); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy') && str_contains($notification->message, 'web-'.$this->service->uuid)); }); test('a failed stop that left the source service down starts it again', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); cloneSafetyMarkServiceRunning($this->service); StopService::shouldRun()->once()->andReturn('docker stop timed out'); StartService::shouldRun()->once()->withArgs(fn (Service $service) => $service->is($this->service)); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy') && str_contains($notification->message, 'started') && str_contains($notification->message, 'docker stop timed out')); }); test('cloning volume data needs the permission to stop the source resource', function () { Bus::fake(); Gate::before(fn ($user, string $ability) => $ability === 'stop' ? false : null); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid); expect(Service::query()->count())->toBe(1); Bus::assertNotDispatched(CallQueuedClosure::class); Livewire::test(CloneMe::class, [ 'project_uuid' => $this->project->uuid, 'environment_uuid' => $this->environment->uuid, ]) ->set('selectedDestination', $this->destination->uuid) ->set('newName', 'cloned-env') ->set('cloneVolumeData', true) ->call('clone', 'environment'); expect(Service::query()->count())->toBe(1) ->and($this->project->environments()->where('name', 'cloned-env')->exists())->toBeFalse(); Bus::assertNotDispatched(CallQueuedClosure::class); }); test('cloning without volume data needs no stop permission and stops nothing', function () { Bus::fake(); Gate::before(fn ($user, string $ability) => $ability === 'stop' ? false : null); Livewire::test(ResourceOperations::class, ['resource' => $this->service]) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); expect(Service::query()->count())->toBe(2); Bus::assertNotDispatched(CallQueuedClosure::class); Bus::assertNotDispatched(VolumeCloneJob::class); }); test('cloning a running database with volume data copies its volumes in one chain between stop and start', function (string $component) { Bus::fake(); $database = StandalonePostgresql::create([ 'name' => 'orders-db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'environment_id' => $this->environment->id, 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(), ]); $database->forceFill(['status' => 'running:healthy'])->saveQuietly(); LocalPersistentVolume::query()->where('resource_type', $database->getMorphClass())->where('resource_id', $database->id)->delete(); foreach (['postgres-data-'.$database->uuid, $database->uuid.'-extra'] as $name) { LocalPersistentVolume::create([ 'name' => $name, 'mount_path' => '/data/'.$name, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), ]); } $this->service->delete(); if ($component === ResourceOperations::class) { Livewire::test(ResourceOperations::class, ['resource' => $database]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); } else { Livewire::test(CloneMe::class, [ 'project_uuid' => $this->project->uuid, 'environment_uuid' => $this->environment->uuid, ]) ->set('selectedDestination', $this->destination->uuid) ->set('newName', 'cloned-env') ->set('cloneVolumeData', true) ->call('clone', 'environment') ->assertHasNoErrors(); } Bus::assertNotDispatched(VolumeCloneJob::class); Bus::assertChained([ CallQueuedClosure::class, VolumeCloneJob::class, VolumeCloneJob::class, CallQueuedClosure::class, ]); })->with([ 'resource operations' => [ResourceOperations::class], 'environment clone' => [CloneMe::class], ]); test('a restored same-server volume clone job has explicit Horizon tags', function () { $server = $this->destination->server; $volume = LocalPersistentVolume::create(['name' => 'clone-target', 'mount_path' => '/data', 'resource_id' => 1, 'resource_type' => Service::class]); // Without explicit tags, Horizon compares the job's models with `==`. Two restored copies of one // server fail that comparison on PHP 8.4 ("Nesting level too deep"), so the chain stops before the copy. $restoredJob = unserialize(serialize(new VolumeCloneJob('clone-source', 'clone-target', $server, $server, $volume))); expect(Tags::extractExplicitTags($restoredJob))->toBe([Server::class.':'.$server->id, LocalPersistentVolume::class.':'.$volume->id]); }); function cloneSafetyCreateDatabase(object $test, string $status): StandalonePostgresql { $database = StandalonePostgresql::create([ 'name' => 'api-db', 'postgres_user' => 'postgres', 'postgres_password' => 'password', 'postgres_db' => 'db', 'image' => 'postgres:17', 'environment_id' => $test->environment->id, 'destination_id' => $test->destination->id, 'destination_type' => $test->destination->getMorphClass(), ]); $database->forceFill(['status' => $status])->saveQuietly(); return $database; } function cloneSafetyApiHeaders(User $user, int $teamId): array { $plainTextToken = Str::random(40); $token = $user->tokens()->create([ 'name' => 'clone-safety', 'token' => hash('sha256', $plainTextToken), 'abilities' => ['*'], 'team_id' => $teamId, ]); auth()->forgetGuards(); return ['Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken]; } function cloneSafetyCreateApplication(object $test, string $status): Application { $application = Application::factory()->create([ 'environment_id' => $test->environment->id, 'destination_id' => $test->destination->id, 'destination_type' => $test->destination->getMorphClass(), ]); $application->forceFill(['status' => $status])->saveQuietly(); LocalPersistentVolume::create([ 'name' => $application->uuid.'-data', 'mount_path' => '/data', 'resource_id' => $application->id, 'resource_type' => $application->getMorphClass(), ]); return $application; } test('a failed volume copy of a running database cloned through the API starts it again and notifies the team', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); cloneSafetyFailVolumeCopy(); $database = cloneSafetyCreateDatabase($this, 'running:healthy'); StopDatabase::shouldRun()->once()->withArgs(fn ($source) => $source->is($database)); StartDatabase::shouldRun()->once()->withArgs(fn ($source) => $source->is($database)); $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id)) ->postJson("/api/v1/databases/{$database->uuid}/clone", [ 'destination_uuid' => $this->destination->uuid, 'clone_volumes' => true, ]) ->assertCreated(); cloneSafetyWorkQueue(); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => $notification->success === false && str_contains($notification->message, 'api-db') && str_contains($notification->message, 'No space left on device')); }); test('a failed stop of a running database cloned through the API copies no data and leaves it running', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); $database = cloneSafetyCreateDatabase($this, 'running:healthy'); cloneSafetyListRunningContainers($database->uuid); StopDatabase::shouldRun()->once()->andReturn('Database stop failed: Connection refused'); StartDatabase::shouldRun()->never(); $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id)) ->postJson("/api/v1/databases/{$database->uuid}/clone", [ 'destination_uuid' => $this->destination->uuid, 'clone_volumes' => true, ]) ->assertCreated(); cloneSafetyWorkQueue(); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy') && str_contains($notification->message, 'api-db') && str_contains($notification->message, 'Connection refused')); }); test('cloning a running database through the API copies its volumes after a successful stop', function () { cloneSafetyUseDatabaseQueue(); $database = cloneSafetyCreateDatabase($this, 'running:healthy'); StopDatabase::shouldRun()->once()->andReturn('Database stopped successfully'); StartDatabase::shouldRun()->once(); $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id)) ->postJson("/api/v1/databases/{$database->uuid}/clone", [ 'destination_uuid' => $this->destination->uuid, 'clone_volumes' => true, ]) ->assertCreated(); cloneSafetyWorkQueue(); Process::assertRan(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); expect(DB::table('failed_jobs')->count())->toBe(0); Notification::assertNothingSent(); }); test('cloning a stopped database through the API copies its volumes without stopping or starting it', function () { Bus::fake(); $database = cloneSafetyCreateDatabase($this, 'exited'); $this->withHeaders(cloneSafetyApiHeaders($this->user, $this->team->id)) ->postJson("/api/v1/databases/{$database->uuid}/clone", [ 'destination_uuid' => $this->destination->uuid, 'clone_volumes' => true, ]) ->assertCreated(); Bus::assertNotDispatched(CallQueuedClosure::class); Bus::assertDispatched(VolumeCloneJob::class); }); /** * `docker ps` lists the running containers: before the stop the given ones, after it the remaining ones. * * @param list $before * @param list $after */ function cloneSafetyListRunningContainersBeforeAndAfterStop(array $before, array $after): void { $listings = 0; Process::fake(function (PendingProcess $process) use (&$listings, $before, $after) { if (! str_contains(remoteProcessScript($process), 'docker ps --filter')) { return Process::result(output: ''); } return Process::result(output: implode("\n", $listings++ === 0 ? $before : $after)); }); } function cloneSafetyCloneApplicationWithVolumes(object $test, Application $application): void { $test->service->delete(); Livewire::test(ResourceOperations::class, ['resource' => $application]) ->set('cloneVolumeData', true) ->call('cloneTo', $test->destination->uuid) ->assertHasNoErrors(); cloneSafetyWorkQueue(); } test('a failed stop that took down some containers of the source application redeploys the commit that ran before', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); $application = cloneSafetyCreateApplication($this, 'running:healthy'); $runningCommit = str_repeat('a', 40); foreach ([$runningCommit => now()->subHour(), str_repeat('b', 40) => now()->subDay()] as $commit => $updatedAt) { ApplicationDeploymentQueue::create([ 'application_id' => $application->id, 'deployment_uuid' => new_public_id(), 'pull_request_id' => 0, 'commit' => $commit, 'status' => 'finished', 'server_id' => $this->destination->server_id, 'destination_id' => $this->destination->id, ])->forceFill(['updated_at' => $updatedAt])->saveQuietly(); } cloneSafetyListRunningContainersBeforeAndAfterStop(['web-'.$application->uuid, 'worker-'.$application->uuid], ['worker-'.$application->uuid]); StopApplication::shouldRun()->once()->andReturn('docker stop failed for worker'); cloneSafetyCloneApplicationWithVolumes($this, $application); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); $deployments = ApplicationDeploymentQueue::query()->where('application_id', $application->id)->orderBy('id')->get(); expect($deployments)->toHaveCount(3); $redeploy = $deployments->last(); expect($redeploy->commit)->toBe($runningCommit) ->and($redeploy->pull_request_id)->toBe(0); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => $notification->success === false && str_contains($notification->message, 'did not copy') && str_contains($notification->message, "started a redeploy of '{$application->name}'") && str_contains($notification->message, 'docker stop failed for worker')); }); test('a failed stop that took down containers of a source application without a known commit redeploys its configured commit', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); $application = cloneSafetyCreateApplication($this, 'running:healthy'); cloneSafetyListRunningContainersBeforeAndAfterStop(['web-'.$application->uuid, 'worker-'.$application->uuid], ['worker-'.$application->uuid]); StopApplication::shouldRun()->once()->andReturn(null); cloneSafetyCloneApplicationWithVolumes($this, $application); expect(ApplicationDeploymentQueue::query()->where('application_id', $application->id)->sole()->commit)->toBe('HEAD'); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, "started a redeploy of '{$application->name}'") && str_contains($notification->message, 'could not find the commit')); }); test('a failed stop that took down no container of the source application does not redeploy it', function () { $this->team->webhookNotificationSettings()->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']); cloneSafetyUseDatabaseQueue(); $application = cloneSafetyCreateApplication($this, 'running:healthy'); cloneSafetyListRunningContainersBeforeAndAfterStop(['web-'.$application->uuid], ['web-'.$application->uuid]); StopApplication::shouldRun()->once()->andReturn('Server is not functional'); cloneSafetyCloneApplicationWithVolumes($this, $application); Process::assertDidntRun(fn ($process) => str_contains(remoteProcessScript($process), 'cp -a')); expect(ApplicationDeploymentQueue::query()->where('application_id', $application->id)->count())->toBe(0); Notification::assertSentTo($this->team, GeneralNotification::class, fn (GeneralNotification $notification) => str_contains($notification->message, 'did not copy') && str_contains($notification->message, 'is still running')); }); test('cloning a running application with volume data copies its volumes in one chain between stop and start', function () { Bus::fake(); $application = cloneSafetyCreateApplication($this, 'running:healthy'); $this->service->delete(); Livewire::test(ResourceOperations::class, ['resource' => $application]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); Bus::assertChained([CallQueuedClosure::class, VolumeCloneJob::class, CallQueuedClosure::class]); expect(ApplicationDeploymentQueue::query()->count())->toBe(0); }); test('cloning a stopped application with volume data neither stops nor redeploys it', function () { Bus::fake(); $application = cloneSafetyCreateApplication($this, 'exited'); $this->service->delete(); Livewire::test(ResourceOperations::class, ['resource' => $application]) ->set('cloneVolumeData', true) ->call('cloneTo', $this->destination->uuid) ->assertHasNoErrors(); Bus::assertNotDispatched(CallQueuedClosure::class); Bus::assertDispatched(VolumeCloneJob::class); expect(ApplicationDeploymentQueue::query()->count())->toBe(0); });