nprasad2077/coolify
1
1 Fork 0
mirror of https://github.com/coollabsio/coolify.git synced 2026-11-04 10:42:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(deployments): skip the compose build when the images match the build inputs

Browse Source 
Compose deployments label each built image with a fingerprint of the build
settings and the resolved build-time variables. A deploy or restart reuses the
commit images when every label matches, so a runtime-only change does not
rebuild.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andras BacsaiandClaude Opus 5.5 committed 2026-10-26 12:58:01 +01:00
1 parent a3843730f8
commit d6c4e773b9
2 files changed
+356 -34
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Jobs/ApplicationDeploymentJob.php
+139 -34
View File
@@ -23,6 +23,7 @@ use App\Models\StandaloneDocker;
use App\Models\SwarmDocker;
use App\Notifications\Application\DeploymentFailed;
use App\Notifications\Application\DeploymentSuccess;
use App\Services\DeploymentConfiguration\ApplicationConfigurationSnapshot;
use App\Support\RemoteSecretReferences;
use App\Support\RemoteSecretValueFormatter;
use App\Support\ValidationPatterns;
@@ -52,6 +53,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
public const BUILD_TIME_ENV_PATH = '/artifacts/build-time.env';
private const BUILD_FINGERPRINT_LABEL = 'coolify.build-fingerprint';
public const BUILD_TIME_SHELL_ENV_PATH = '/artifacts/build-time-shell.env';
public const BUILD_TIME_ENV_LAUNCHER_PATH = '/artifacts/run-with-build-time-env';
@@ -103,6 +106,8 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private bool $restart_only;
private ?string $docker_compose_build_fingerprint = null;
private ?string $dockerImage = null;
private ?string $dockerImageTag = null;
@@ -253,7 +258,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->force_rebuild = true;
}
$this->restart_only = $this->application_deployment_queue->restart_only;
$this->restart_only = $this->restart_only && $this->application->build_pack !== 'dockerimage' && $this->application->build_pack !== 'dockerfile';
// These build packs need the cloned repository to start, so a restart runs a regular deployment.
// The deployment skips the build when the image for the commit exists.
$this->restart_only = $this->restart_only && ! in_array($this->application->build_pack, ['dockerimage', 'dockerfile', 'dockercompose'], true);
$this->only_this_server = $this->application_deployment_queue->only_this_server;
$this->dockerImagePreviewTag = $this->application_deployment_queue->docker_registry_image_tag;
$this->validateDockerRegistryImageConfiguration();
@@ -822,6 +829,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->generate_build_env_variables();
$this->loadComposeFileForDeployment();
$buildEnvironment = null;
if ($this->application->settings->is_raw_compose_deployment_enabled) {
$this->application->oldRawParser();
$yaml = $composeFile = $this->application->docker_compose_raw;
@@ -853,6 +861,10 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$composeFile = $this->add_build_secrets_to_compose($composeFile);
}
$buildEnvironment = $this->generate_buildtime_environment_variables();
$this->docker_compose_build_fingerprint = $this->docker_compose_build_fingerprint($buildEnvironment);
$composeFile = $this->add_build_fingerprint_to_compose(convertToArray($composeFile));
$yaml = Yaml::dump(convertToArray($composeFile), 10);
}
$this->generated_docker_compose = $yaml;
@@ -865,13 +877,55 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
'hidden' => true,
]);
if ($this->can_reuse_docker_compose_images($composeFile)) {
$this->skip_build = true;
} else {
$this->build_docker_compose_images($composeFile, $buildEnvironment);
}
// Save runtime environment variables AFTER the build
// This overwrites the build-time .env with ALL variables (build-time + runtime)
$this->save_runtime_environment_variables();
$this->pull_docker_compose_images();
$this->stop_running_container(force: true);
$this->application_deployment_queue->addLogEntry('Starting new application.');
$networkId = $this->application->uuid;
if ($this->pull_request_id !== 0) {
$networkId = "{$this->application->uuid}-{$this->pull_request_id}";
}
if ($this->server->isSwarm()) {
// TODO
} else {
$this->execute_remote_command([
"docker network inspect '{$networkId}' >/dev/null 2>&1 || docker network create --attachable '{$networkId}' >/dev/null || true",
'hidden' => true,
'ignore_errors' => true,
], [
"docker network connect {$networkId} coolify-proxy >/dev/null 2>&1 || true",
'hidden' => true,
'ignore_errors' => true,
]);
}
$this->start_docker_compose_services();
$this->application_deployment_queue->addLogEntry('New container started.');
}
/**
* Builds the images of the Compose services that have a build section.
*/
private function build_docker_compose_images(mixed $composeFile, ?Collection $buildEnvironment): void
{
// Modify Dockerfiles for ARGs and build secrets
$this->modify_dockerfiles_for_compose($composeFile);
// Build new container to limit downtime.
$this->application_deployment_queue->addLogEntry('Pulling & building required images.');
// Save build-time .env file BEFORE the build
$this->save_buildtime_environment_variables();
$this->save_buildtime_environment_variables($buildEnvironment);
if ($this->docker_compose_custom_build_command) {
// Auto-inject -f (compose file) and --env-file flags using helper function
@@ -935,36 +989,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
[executeInDocker($this->deployment_uuid, $command), 'hidden' => true],
);
}
// Save runtime environment variables AFTER the build
// This overwrites the build-time .env with ALL variables (build-time + runtime)
$this->save_runtime_environment_variables();
$this->pull_docker_compose_images();
$this->stop_running_container(force: true);
$this->application_deployment_queue->addLogEntry('Starting new application.');
$networkId = $this->application->uuid;
if ($this->pull_request_id !== 0) {
$networkId = "{$this->application->uuid}-{$this->pull_request_id}";
}
if ($this->server->isSwarm()) {
// TODO
} else {
$this->execute_remote_command([
"docker network inspect '{$networkId}' >/dev/null 2>&1 || docker network create --attachable '{$networkId}' >/dev/null || true",
'hidden' => true,
'ignore_errors' => true,
], [
"docker network connect {$networkId} coolify-proxy >/dev/null 2>&1 || true",
'hidden' => true,
'ignore_errors' => true,
]);
}
$this->start_docker_compose_services();
$this->application_deployment_queue->addLogEntry('New container started.');
}
/**
@@ -1578,6 +1602,87 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
return false;
}
/**
* Hash of every input that changes a Compose build: the build settings, the resolved build-time
* variables (with domains, shared variables, and secrets), and the variables of the build command.
* The hash goes on the built images as a label, so a later deployment can see what an image was built from.
*/
private function docker_compose_build_fingerprint(Collection $buildEnvironment): string
{
$buildSettings = collect((new ApplicationConfigurationSnapshot($this->application))->toArray()['sections'])
->flatMap(fn (array $section) => $section['items'])
->where('impact', 'build')
->mapWithKeys(fn (array $item) => [$item['key'] => $item['compare_value']]);
return hash('sha256', json_encode([$buildSettings, $buildEnvironment->values(), $this->coolify_variables]));
}
private function add_build_fingerprint_to_compose(array $composeFile): array
{
foreach ($composeFile['services'] ?? [] as $serviceName => $service) {
if (! isset($service['build'])) {
continue;
}
if (is_string($service['build'])) {
$service['build'] = ['context' => $service['build']];
}
$labels = $service['build']['labels'] ?? [];
if (array_is_list($labels)) {
$labels[] = self::BUILD_FINGERPRINT_LABEL.'='.$this->docker_compose_build_fingerprint;
} else {
$labels[self::BUILD_FINGERPRINT_LABEL] = $this->docker_compose_build_fingerprint;
}
$service['build']['labels'] = $labels;
$composeFile['services'][$serviceName] = $service;
}
return $composeFile;
}
/**
* Whether every Compose service with a build section has an image on the server for this commit
* that was built from the same inputs, so the deployment can start it without a build.
* Only images tagged by the parser with the commit qualify: an image name set in the Compose
* file can hold an older build.
*/
private function can_reuse_docker_compose_images(mixed $composeFile): bool
{
if ($this->force_rebuild
|| $this->use_build_server
|| $this->pull_request_id !== 0
|| $this->application->settings->is_raw_compose_deployment_enabled
|| $this->docker_compose_custom_build_command
|| blank($this->docker_compose_build_fingerprint)
|| blank($this->commit)
|| $this->commit === 'HEAD') {
return false;
}
$images = collect(data_get($composeFile, 'services', []))
->filter(fn ($service) => data_get($service, 'build') !== null)
->map(fn ($service, $name) => [data_get($service, 'image'), "{$this->application->uuid}_{$name}:{$this->commit}"]);
if ($images->isEmpty() || $images->contains(fn (array $image) => $image[0] !== $image[1])) {
return false;
}
$imageNames = $images->map(fn (array $image) => escapeshellarg($image[1]))->implode(' ');
$format = escapeshellarg('{{ index .Config.Labels "'.self::BUILD_FINGERPRINT_LABEL.'" }}');
$this->execute_remote_command([
"docker image inspect --format {$format} {$imageNames} 2>/dev/null",
'hidden' => true,
'ignore_errors' => true,
'save' => 'compose_image_fingerprints',
]);
$fingerprints = $this->trimmedSavedOutput('compose_image_fingerprints')->explode("\n")->map(fn (string $line) => trim($line));
if ($fingerprints->count() !== $images->count() || $fingerprints->contains(fn (string $fingerprint) => $fingerprint !== $this->docker_compose_build_fingerprint)) {
return false;
}
$this->application_deployment_queue->addLogEntry('Images found with the same Git Commit SHA and build inputs. Build step skipped.');
return true;
}
private function has_remote_buildtime_secret_references(): bool
{
$environmentVariables = $this->pull_request_id === 0
@@ -2587,9 +2692,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
return $variables->reject(fn ($value, $key): bool => str_contains((string) $key, '.'));
}
private function save_buildtime_environment_variables()
private function save_buildtime_environment_variables(?Collection $environment_variables = null)
{
$environment_variables = $this->generate_buildtime_environment_variables();
$environment_variables ??= $this->generate_buildtime_environment_variables();
[$shell_environment_variables, $dotted_environment_variables] = $environment_variables->partition(function (string $environmentVariable): bool {
[$key] = explode('=', $environmentVariable, 2);
tests/Feature/ComposeImageReuseTest.php
+217
View File
@@ -0,0 +1,217 @@
<?php
use App\Jobs\ApplicationDeploymentJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\EnvironmentVariable;
use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Process;
uses(RefreshDatabase::class);
const REUSE_COMMIT = '0123456789abcdef0123456789abcdef01234567';
const REUSE_FINGERPRINT = 'current-fingerprint';
class ComposeImageReuseDeploymentJob extends ApplicationDeploymentJob
{
/** @var array<int, string> */
public array $commands = [];
/** @var array<int, string> */
public array $logEntries = [];
public string $inspectOutput = REUSE_FINGERPRINT."\n".REUSE_FINGERPRINT;
public function __construct() {}
public function execute_remote_command(...$commands)
{
$savedOutputs = (new ReflectionProperty(ApplicationDeploymentJob::class, 'saved_outputs'))->getValue($this);
foreach ($commands as $command) {
$this->commands[] = $command[0];
if (isset($command['save'])) {
$savedOutputs->put($command['save'], $this->inspectOutput);
}
}
}
}
function composeReuseApplication(string $compose): Application
{
return Application::factory()->create([
'build_pack' => 'dockercompose',
'docker_compose_raw' => $compose,
'environment_id' => test()->environment->id,
'destination_id' => test()->destination->id,
'destination_type' => test()->destination->getMorphClass(),
])->fresh();
}
function composeReuseJob(Application $application, array $properties = []): ComposeImageReuseDeploymentJob
{
$job = new ComposeImageReuseDeploymentJob;
$queue = Mockery::mock(ApplicationDeploymentQueue::class);
$queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message) use ($job) {
$job->logEntries[] = $message;
});
foreach (array_merge([
'application' => $application,
'application_deployment_queue' => $queue,
'saved_outputs' => collect(),
'commit' => REUSE_COMMIT,
'pull_request_id' => 0,
'force_rebuild' => false,
'use_build_server' => false,
'docker_compose_custom_build_command' => null,
'docker_compose_build_fingerprint' => REUSE_FINGERPRINT,
'coolify_variables' => 'COOLIFY_RESOURCE_UUID=abc ',
], $properties) as $property => $value) {
(new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($job, $value);
}
return $job;
}
function canReuseComposeImages(ComposeImageReuseDeploymentJob $job, Application $application): bool
{
$composeFile = $application->parse(commit: REUSE_COMMIT);
return (new ReflectionMethod(ApplicationDeploymentJob::class, 'can_reuse_docker_compose_images'))->invoke($job, $composeFile);
}
function composeBuildFingerprint(Application $application, array $buildEnvironment = ['APP_URL=https://a.example.com']): string
{
return (new ReflectionMethod(ApplicationDeploymentJob::class, 'docker_compose_build_fingerprint'))
->invoke(composeReuseJob($application->fresh()), collect($buildEnvironment));
}
function addComposeEnvironmentVariable(Application $application, string $key, bool $isBuildtime): void
{
EnvironmentVariable::create([
'key' => $key,
'value' => 'value',
'is_buildtime' => $isBuildtime,
'is_runtime' => true,
'resourceable_type' => Application::class,
'resourceable_id' => $application->id,
]);
}
beforeEach(function () {
InstanceSettings::forceCreate(['id' => 0]);
Process::fake(['*' => Process::result(output: '')]);
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
]);
$this->destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
$this->environment = Project::factory()->create(['team_id' => $team->id])->environments()->first();
$this->compose = "services:\n web:\n build: .\n worker:\n build: ./worker\n cache:\n image: redis:7\n";
});
it('reuses the commit images when every image was built from the current inputs', function () {
$application = composeReuseApplication($this->compose);
$job = composeReuseJob($application);
expect(canReuseComposeImages($job, $application))->toBeTrue()
->and($job->commands)->toBe([
"docker image inspect --format '{{ index .Config.Labels \"coolify.build-fingerprint\" }}' '{$application->uuid}_web:".REUSE_COMMIT."' '{$application->uuid}_worker:".REUSE_COMMIT."' 2>/dev/null",
])
->and($job->logEntries)->toBe(['Images found with the same Git Commit SHA and build inputs. Build step skipped.']);
});
it('builds when an image was built from other inputs', function () {
$application = composeReuseApplication($this->compose);
$job = composeReuseJob($application);
$job->inspectOutput = REUSE_FINGERPRINT."\nold-fingerprint";
expect(canReuseComposeImages($job, $application))->toBeFalse();
});
it('builds when an image has no fingerprint label', function () {
$application = composeReuseApplication($this->compose);
$job = composeReuseJob($application);
$job->inspectOutput = REUSE_FINGERPRINT."\n";
expect(canReuseComposeImages($job, $application))->toBeFalse();
});
it('builds when an image is missing on the server', function () {
$application = composeReuseApplication($this->compose);
$job = composeReuseJob($application);
$job->inspectOutput = REUSE_FINGERPRINT;
expect(canReuseComposeImages($job, $application))->toBeFalse();
});
it('builds when a built service sets its own image name', function () {
$application = composeReuseApplication("services:\n web:\n build: .\n image: example/web:latest\n");
$job = composeReuseJob($application);
expect(canReuseComposeImages($job, $application))->toBeFalse()
->and($job->commands)->toBe([]);
});
it('builds without checking images when the deployment cannot reuse a commit image', function (array $properties) {
$application = composeReuseApplication($this->compose);
$job = composeReuseJob($application, $properties);
expect(canReuseComposeImages($job, $application))->toBeFalse()
->and($job->commands)->toBe([]);
})->with([
'forced rebuild' => [['force_rebuild' => true]],
'unresolved commit' => [['commit' => 'HEAD']],
'custom build command' => [['docker_compose_custom_build_command' => 'docker compose build']],
'build server' => [['use_build_server' => true]],
'no fingerprint' => [['docker_compose_build_fingerprint' => null]],
]);
it('keeps the fingerprint when only a runtime variable changes', function () {
$application = composeReuseApplication($this->compose);
$before = composeBuildFingerprint($application);
addComposeEnvironmentVariable($application, 'RUNTIME_ONLY', isBuildtime: false);
expect(composeBuildFingerprint($application))->toBe($before);
});
it('changes the fingerprint when a build input changes', function (Closure $change, array $buildEnvironment) {
$application = composeReuseApplication($this->compose);
$before = composeBuildFingerprint($application);
$change($application);
expect(composeBuildFingerprint($application, $buildEnvironment))->not->toBe($before);
})->with([
'build-time variable' => [fn (Application $application) => addComposeEnvironmentVariable($application, 'BUILD_VALUE', isBuildtime: true), ['APP_URL=https://a.example.com']],
'build setting' => [fn (Application $application) => $application->settings()->update(['inject_build_args_to_dockerfile' => false]), ['APP_URL=https://a.example.com']],
'resolved build value such as a domain or shared variable' => [fn () => null, ['APP_URL=https://b.example.com']],
]);
it('adds the fingerprint label to every service with a build section', function () {
$job = composeReuseJob(composeReuseApplication($this->compose));
$composeFile = (new ReflectionMethod(ApplicationDeploymentJob::class, 'add_build_fingerprint_to_compose'))->invoke($job, [
'services' => [
'web' => ['build' => '.'],
'api' => ['build' => ['context' => './api', 'labels' => ['team' => 'core']]],
'worker' => ['build' => ['context' => './worker', 'labels' => ['team=core']]],
'cache' => ['image' => 'redis:7'],
],
]);
expect($composeFile['services'])->toBe([
'web' => ['build' => ['context' => '.', 'labels' => ['coolify.build-fingerprint='.REUSE_FINGERPRINT]]],
'api' => ['build' => ['context' => './api', 'labels' => ['team' => 'core', 'coolify.build-fingerprint' => REUSE_FINGERPRINT]]],
'worker' => ['build' => ['context' => './worker', 'labels' => ['team=core', 'coolify.build-fingerprint='.REUSE_FINGERPRINT]]],
'cache' => ['image' => 'redis:7'],
]);
});