enhance(config): add container.sweep option (#1275)

The Docker proxy labels every container, network and volume a job creates and removes them when the job ends, which breaks jobs that deploy long-running services to the runner's own Docker daemon. Document that behavior and add `container.sweep` (default `true`) so operators who deploy to the same daemon can opt out. Kept resources stay labeled with `com.gitea.runner.job`.

Fixes: https://gitea.com/gitea/runner/issues/1243
Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1275
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
Co-authored-by: bircni <bircni@icloud.com>

README.md

@@ -264,7 +264,7 @@ volumes:
   - ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
 ```
 
-Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. The proxy removes containers, networks and volumes after post steps and the completed hook. Named volumes created through it are job-scoped. To retain resources, mount the daemon socket explicitly in `container.options`. Host jobs use their existing Docker access, where `.` works.
+Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
 
 #### Proxy
 

act/runner/run_context.go

@@ -725,7 +725,7 @@ func (rc *RunContext) cleanupJobResources(networkName string, createAndDeleteNet
 			logger.Infof("Cleaning up services for job %s", rc.JobName)
 			errs = append(errs, rc.stopServiceContainers()(ctx))
 		}
-		if !common.Dryrun(ctx) && (preclean || rc.hadDockerProxy) {
+		if !common.Dryrun(ctx) && !rc.Config.NoSweep && (preclean || rc.hadDockerProxy) {
 			errs = append(errs, container.RemoveDockerJobResources(ctx, rc.jobContainerName()))
 		}
 		if preclean || rc.deferVolumeCleanup == nil {

act/runner/run_context_test.go

@@ -778,9 +778,10 @@ func TestRemoveDockerActionImagesByTag(t *testing.T) {
 func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
 	t.Setenv("TMPDIR", "/tmp")
 	proxyDir := t.TempDir()
-	for _, name := range []string{"synchronous", "proxy", "closed proxy", "deferred", "preclean deferred", "canceled"} {
+	for _, name := range []string{"synchronous", "proxy", "closed proxy", "kept proxy", "deferred", "preclean deferred", "canceled"} {
 		t.Run(name, func(t *testing.T) {
 			proxy, preclean, deferred := strings.HasSuffix(name, "proxy"), strings.HasPrefix(name, "preclean"), strings.HasSuffix(name, "deferred")
+			removesLabelled := (proxy || preclean) && name != "kept proxy"
 			if proxy && runtime.GOOS == "windows" {
 				t.Skip("Unix socket ownership is unavailable on Windows")
 			}
@@ -790,7 +791,7 @@ func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
 			service.On("Remove").Return(func(context.Context) error { return removeError }).Once()
 			service.On("Close").Return(func(context.Context) error { return closeError }).Once()
 			rc := &RunContext{
-				Config:            &Config{CacheContainer: "cache-container"},
+				Config:            &Config{CacheContainer: "cache-container", NoSweep: name == "kept proxy"},
 				Run:               &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
 				JobContainer:      job,
 				serviceContainers: []*serviceContainer{{name: "svc", container: service}},
@@ -846,8 +847,8 @@ func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
 				}
 				assert.Equal(t, 2, volumeRemovals)
 			}
-			assert.Equal(t, proxy || preclean, strings.Contains(err.Error(), "labelled GET /containers/json"))
-			if proxy || preclean {
+			assert.Equal(t, removesLabelled, strings.Contains(err.Error(), "labelled GET /containers/json"))
+			if removesLabelled {
 				require.ErrorContains(t, err, "labelled GET /networks")
 				require.ErrorContains(t, err, "labelled GET /volumes")
 			}

act/runner/runner.go

@@ -87,6 +87,7 @@ type Config struct {
 	MaxParallel                       int           // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
 	AllocatePTY                       bool          // allocate a pseudo-TTY for each step's process
 	ServiceReadyTimeout               time.Duration // how long a job waits for its service containers to report healthy (0 uses the default)
+	NoSweep                           bool          // keep the containers, networks and volumes a job creates through its docker proxy
 	RunnerName                        string        // name this runner registered with, reported as `runner.name`, defaults to the hostname
 	JobStartedHook                    string        // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
 	JobCompletedHook                  string        // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty

internal/app/run/runner.go

@@ -617,6 +617,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
 		},
 		ContainerOptions:                  r.cfg.Container.Options,
 		ServiceReadyTimeout:               r.cfg.Container.ServiceReadyTimeout,
+		NoSweep:                           r.cfg.Container.Sweep != nil && !*r.cfg.Container.Sweep,
 		ContainerDaemonSocket:             r.cfg.Container.DockerHost,
 		Privileged:                        r.cfg.Container.Privileged,
 		DefaultActionInstance:             r.getDefaultActionsURL(task),

internal/pkg/config/config.example.yaml

@@ -285,6 +285,9 @@ container:
   # How long a job waits for a service container that declares a healthcheck to become
   # healthy. A negative value (e.g. -1s) starts the steps without waiting.
   #service_ready_timeout: 5m
+  # Remove the containers, networks and volumes a job creates through its Docker proxy when
+  # the job ends. Set to false when jobs deploy services to the runner's Docker daemon.
+  #sweep: true
 
 host:
   # The parent directory of a job's working directory.

internal/pkg/config/config.go

@@ -181,6 +181,7 @@ type Container struct {
 	DockerTimeout        time.Duration                 `yaml:"docker_timeout"`         // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or runner
 	BindWorkdir          bool                          `yaml:"bind_workdir"`           // BindWorkdir mounts the workspace from a host directory instead of a Docker volume.
 	ServiceReadyTimeout  time.Duration                 `yaml:"service_ready_timeout"`  // ServiceReadyTimeout bounds how long a job waits for a service container that declares a healthcheck to report healthy. Negative disables waiting.
+	Sweep                *bool                         `yaml:"sweep"`                  // Sweep removes the containers, networks and volumes a job creates through its Docker proxy when the job ends. Unset means enabled.
 }
 
 // Values of Runner.ToolCacheMode: the runner mounts no tool cache, or one that every job reuses.