mirror of
https://github.com/coollabsio/coolify.git
synced 2026-11-04 10:42:23 +00:00
feat(proxy): retry failed connections on another container
Add the Traefik retry middleware and the Caddy lb_try_duration to the
application proxy labels again, so a request that fails on the old
container during a rolling update goes to another container (#8627).
main removed them in 5dacaed9f for v4.5.2, because the label change
gives a 503 window on the first rolling update after a settings save.
This commit is contained in:
Andras Bacsai committed
1 parent
71f1c33bf1
commit
a0b1c4ff02
2 files changed
+36
-10
No files matched your search
@@ -801,7 +801,7 @@ function caddySiteAddress(string $scheme, string $host, bool $isForceHttpsEnable
|
||||
return "{$scheme}://{$host}";
|
||||
}
|
||||
|
||||
function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false)
|
||||
function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false)
|
||||
{
|
||||
$labels = collect([]);
|
||||
if ($serviceLabels) {
|
||||
@@ -854,6 +854,11 @@ function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains,
|
||||
} else {
|
||||
$labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy={{upstreams}}");
|
||||
}
|
||||
if ($is_retry_enabled) {
|
||||
// A failed connection, for example to the old container that stops during a rolling
|
||||
// update, is retried on another upstream. Caddy always retries a connection that failed.
|
||||
$labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy.lb_try_duration=5s");
|
||||
}
|
||||
$labels->push("caddy_{$loop}.{$handle}={$path}*");
|
||||
if ($is_gzip_enabled) {
|
||||
$labels->push("caddy_{$loop}.encode=zstd gzip");
|
||||
@@ -958,7 +963,7 @@ function traefikHostRulesWithoutIpv6Brackets(Collection $labels): Collection
|
||||
) : $label);
|
||||
}
|
||||
|
||||
function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false)
|
||||
function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false)
|
||||
{
|
||||
$labels = collect([]);
|
||||
$labels->push('traefik.enable=true');
|
||||
@@ -1013,6 +1018,14 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_
|
||||
->filter()
|
||||
->unique();
|
||||
}
|
||||
if ($is_retry_enabled) {
|
||||
// Sends a request to another container when the connection fails, for example to the old
|
||||
// container that stops during a rolling update. Traefik does not retry after a response.
|
||||
// A retry can pick the old container again under load; 3 attempts left about 5% of the
|
||||
// requests failing in a live test, 5 attempts less than 1% (#8627).
|
||||
$labels->push("traefik.http.middlewares.retry-{$uuid}.retry.attempts=5");
|
||||
$middlewares_from_labels->push("retry-{$uuid}");
|
||||
}
|
||||
foreach ($domains as $loop => $domain) {
|
||||
try {
|
||||
if ($generate_unique_uuid) {
|
||||
@@ -1278,6 +1291,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
|
||||
http_basic_auth_bcrypt_cost: $application->http_basic_auth_bcrypt_cost,
|
||||
noindex_domains: $noindexDomains,
|
||||
domainPortOverrides: $application->domain_port_overrides ?? [],
|
||||
is_retry_enabled: true,
|
||||
certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver),
|
||||
));
|
||||
break;
|
||||
@@ -1302,6 +1316,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
|
||||
supports_log_append: $application->destination->server->caddySupportsLogAppend(),
|
||||
supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(),
|
||||
domainPortOverrides: $application->domain_port_overrides ?? [],
|
||||
is_retry_enabled: true,
|
||||
));
|
||||
break;
|
||||
}
|
||||
@@ -1322,6 +1337,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
|
||||
noindex_domains: $noindexDomains,
|
||||
escape_redirect_replacement_for_compose: false,
|
||||
domainPortOverrides: $application->domain_port_overrides ?? [],
|
||||
is_retry_enabled: true,
|
||||
certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver),
|
||||
));
|
||||
$labels = $labels->merge(fqdnLabelsForCaddy(
|
||||
@@ -1344,6 +1360,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
|
||||
supports_log_append: $application->destination->server->caddySupportsLogAppend(),
|
||||
supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(),
|
||||
domainPortOverrides: $application->domain_port_overrides ?? [],
|
||||
is_retry_enabled: true,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,18 +110,27 @@ function runRollingUpdateDrainStep(Application $application, bool $newVersionIsH
|
||||
return $logs;
|
||||
}
|
||||
|
||||
it('keeps the application proxy labels of v4.5.0 without a retry, so a redeploy after an upgrade does not change them', function (string $proxyType, bool $exactLabels) {
|
||||
$application = createApplicationForRollingUpdateDrainTest($proxyType, $exactLabels);
|
||||
it('retries failed connections on another container in Traefik application labels', function (bool $exactLabels) {
|
||||
$application = createApplicationForRollingUpdateDrainTest(ProxyTypes::TRAEFIK->value, $exactLabels);
|
||||
|
||||
expect(implode("\n", generateLabelsApplication($application)))
|
||||
->not->toContain('retry')
|
||||
->not->toContain('lb_try_duration');
|
||||
$labels = collect(generateLabelsApplication($application));
|
||||
|
||||
expect($labels)->toContain("traefik.http.middlewares.retry-{$application->uuid}.retry.attempts=5");
|
||||
$routerMiddlewares = $labels->filter(fn (string $label) => str_contains($label, "routers.https-0-{$application->uuid}.middlewares="));
|
||||
expect($routerMiddlewares)->toHaveCount(1)
|
||||
->and($routerMiddlewares->first())->toContain("retry-{$application->uuid}");
|
||||
})->with([
|
||||
'Traefik generic labels' => [ProxyTypes::TRAEFIK->value, false],
|
||||
'Traefik exact labels' => [ProxyTypes::TRAEFIK->value, true],
|
||||
'Caddy exact labels' => [ProxyTypes::CADDY->value, true],
|
||||
'generic labels' => false,
|
||||
'exact labels' => true,
|
||||
]);
|
||||
|
||||
it('retries failed connections on another upstream in Caddy application labels', function () {
|
||||
$application = createApplicationForRollingUpdateDrainTest(ProxyTypes::CADDY->value, exactLabels: true);
|
||||
|
||||
expect(generateLabelsApplication($application))
|
||||
->toContain('caddy_0.handle_path.0_reverse_proxy.lb_try_duration=5s');
|
||||
});
|
||||
|
||||
it('does not add retry labels to preview deployments', function () {
|
||||
$application = createApplicationForRollingUpdateDrainTest();
|
||||
$preview = ApplicationPreview::forceCreate([
|
||||
|
||||
Reference in new issue
Block a user