diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index d3ca519766..f179df1d96 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -801,7 +801,7 @@ function caddySiteAddress(string $scheme, string $host, bool $isForceHttpsEnable return "{$scheme}://{$host}"; } -function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false) +function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false) { $labels = collect([]); if ($serviceLabels) { @@ -854,6 +854,11 @@ function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, } else { $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy={{upstreams}}"); } + if ($is_retry_enabled) { + // A failed connection, for example to the old container that stops during a rolling + // update, is retried on another upstream. Caddy always retries a connection that failed. + $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy.lb_try_duration=5s"); + } $labels->push("caddy_{$loop}.{$handle}={$path}*"); if ($is_gzip_enabled) { $labels->push("caddy_{$loop}.encode=zstd gzip"); @@ -958,7 +963,7 @@ function traefikHostRulesWithoutIpv6Brackets(Collection $labels): Collection ) : $label); } -function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false) +function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false) { $labels = collect([]); $labels->push('traefik.enable=true'); @@ -1013,6 +1018,14 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ ->filter() ->unique(); } + if ($is_retry_enabled) { + // Sends a request to another container when the connection fails, for example to the old + // container that stops during a rolling update. Traefik does not retry after a response. + // A retry can pick the old container again under load; 3 attempts left about 5% of the + // requests failing in a live test, 5 attempts less than 1% (#8627). + $labels->push("traefik.http.middlewares.retry-{$uuid}.retry.attempts=5"); + $middlewares_from_labels->push("retry-{$uuid}"); + } foreach ($domains as $loop => $domain) { try { if ($generate_unique_uuid) { @@ -1278,6 +1291,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview http_basic_auth_bcrypt_cost: $application->http_basic_auth_bcrypt_cost, noindex_domains: $noindexDomains, domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver), )); break; @@ -1302,6 +1316,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview supports_log_append: $application->destination->server->caddySupportsLogAppend(), supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(), domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, )); break; } @@ -1322,6 +1337,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview noindex_domains: $noindexDomains, escape_redirect_replacement_for_compose: false, domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver), )); $labels = $labels->merge(fqdnLabelsForCaddy( @@ -1344,6 +1360,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview supports_log_append: $application->destination->server->caddySupportsLogAppend(), supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(), domainPortOverrides: $application->domain_port_overrides ?? [], + is_retry_enabled: true, )); } } diff --git a/tests/Feature/RollingUpdateProxyDrainTest.php b/tests/Feature/RollingUpdateProxyDrainTest.php index 2ad2e355d3..4ad85d0d63 100644 --- a/tests/Feature/RollingUpdateProxyDrainTest.php +++ b/tests/Feature/RollingUpdateProxyDrainTest.php @@ -110,18 +110,27 @@ function runRollingUpdateDrainStep(Application $application, bool $newVersionIsH return $logs; } -it('keeps the application proxy labels of v4.5.0 without a retry, so a redeploy after an upgrade does not change them', function (string $proxyType, bool $exactLabels) { - $application = createApplicationForRollingUpdateDrainTest($proxyType, $exactLabels); +it('retries failed connections on another container in Traefik application labels', function (bool $exactLabels) { + $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::TRAEFIK->value, $exactLabels); - expect(implode("\n", generateLabelsApplication($application))) - ->not->toContain('retry') - ->not->toContain('lb_try_duration'); + $labels = collect(generateLabelsApplication($application)); + + expect($labels)->toContain("traefik.http.middlewares.retry-{$application->uuid}.retry.attempts=5"); + $routerMiddlewares = $labels->filter(fn (string $label) => str_contains($label, "routers.https-0-{$application->uuid}.middlewares=")); + expect($routerMiddlewares)->toHaveCount(1) + ->and($routerMiddlewares->first())->toContain("retry-{$application->uuid}"); })->with([ - 'Traefik generic labels' => [ProxyTypes::TRAEFIK->value, false], - 'Traefik exact labels' => [ProxyTypes::TRAEFIK->value, true], - 'Caddy exact labels' => [ProxyTypes::CADDY->value, true], + 'generic labels' => false, + 'exact labels' => true, ]); +it('retries failed connections on another upstream in Caddy application labels', function () { + $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::CADDY->value, exactLabels: true); + + expect(generateLabelsApplication($application)) + ->toContain('caddy_0.handle_path.0_reverse_proxy.lb_try_duration=5s'); +}); + it('does not add retry labels to preview deployments', function () { $application = createApplicationForRollingUpdateDrainTest(); $preview = ApplicationPreview::forceCreate([