fix(cleanup): prune audit events in batches and delete old finished runner executions

app/Console/Commands/CleanupDatabase.php

@@ -2,8 +2,11 @@
 
 namespace App\Console\Commands;
 
+use App\Enums\GithubRunnerStatus;
 use App\Models\AuditEvent;
+use Closure;
 use Illuminate\Console\Command;
+use Illuminate\Database\Query\Builder;
 use Illuminate\Support\Facades\DB;
 
 class CleanupDatabase extends Command
@@ -43,22 +46,29 @@ class CleanupDatabase extends Command
         $this->cleanupTable('application_deployment_queues', 'created_at', now()->subDays($keep_days));
 
         $this->cleanupTable('scheduled_task_executions', 'created_at', now()->subDays($keep_days));
+
+        $this->cleanupTable('github_runner_executions', 'created_at', now()->subDays($keep_days), fn (Builder $query) => $query
+            ->whereNotIn('status', array_map(fn (GithubRunnerStatus $status): string => $status->value, GithubRunnerStatus::active())));
     }
 
     /**
      * Delete the rows older than the given value in batches, so a large table is not locked
      * by one long DELETE.
+     *
+     * @param  (Closure(Builder): mixed)|null  $constraint  Limits the rows that may be deleted.
      */
-    private function cleanupTable(string $table, string $column, mixed $olderThan): void
+    private function cleanupTable(string $table, string $column, mixed $olderThan, ?Closure $constraint = null): void
     {
-        $count = DB::table($table)->where($column, '<', $olderThan)->count();
+        $query = fn (): Builder => DB::table($table)->where($column, '<', $olderThan)->when($constraint, $constraint);
+
+        $count = $query()->count();
         $this->line("Delete $count entries from $table.");
         if (! $this->option('yes')) {
             return;
         }
 
         do {
-            $deleted = DB::table($table)->where($column, '<', $olderThan)->limit(self::DELETE_BATCH_SIZE)->delete();
+            $deleted = $query()->limit(self::DELETE_BATCH_SIZE)->delete();
         } while ($deleted > 0);
     }
 }

app/Models/AuditEvent.php

@@ -41,6 +41,8 @@ class AuditEvent extends Model
         'configuration_snapshot', 'configuration_diff', 'content', 'file_storage_content',
     ];
 
+    private const PRUNE_BATCH_SIZE = 1000;
+
     protected $fillable = [
         'team_id',
         'event',
@@ -228,11 +230,23 @@ class AuditEvent extends Model
             ->first()?->team()?->id;
     }
 
+    /**
+     * Deletes in batches, so a large table is not locked by one long DELETE.
+     */
     public static function pruneExpired(): int
     {
-        return self::query()
-            ->where('created_at', '<', now()->subDays(90))
-            ->delete();
+        $olderThan = now()->subDays(90);
+        $total = 0;
+
+        do {
+            $deleted = self::query()
+                ->where('created_at', '<', $olderThan)
+                ->limit(self::PRUNE_BATCH_SIZE)
+                ->delete();
+            $total += $deleted;
+        } while ($deleted > 0);
+
+        return $total;
     }
 
     /**

tests/Feature/CleanupDatabaseRunnerExecutionsTest.php

@@ -0,0 +1,96 @@
+<?php
+
+use App\Enums\GithubRunnerStatus;
+use App\Models\AuditEvent;
+use App\Models\GithubApp;
+use App\Models\GithubRunnerExecution;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\DB;
+
+uses(RefreshDatabase::class);
+
+function cleanupRunnerTestExecution(GithubApp $githubApp, GithubRunnerStatus $status, int $ageInDays): GithubRunnerExecution
+{
+    static $jobId = 9000;
+
+    $execution = GithubRunnerExecution::create([
+        'github_app_id' => $githubApp->id,
+        'trigger_workflow_job_id' => ++$jobId,
+        'status' => $status,
+    ]);
+    GithubRunnerExecution::query()->whereKey($execution->id)->toBase()->update(['created_at' => now()->subDays($ageInDays)]);
+
+    return $execution;
+}
+
+beforeEach(function () {
+    $team = Team::factory()->create();
+    $this->githubApp = GithubApp::create([
+        'name' => 'cleanup-runner-app',
+        'api_url' => 'https://api.github.com',
+        'html_url' => 'https://github.com',
+        'custom_user' => 'git',
+        'custom_port' => 22,
+        'team_id' => $team->id,
+        'is_system_wide' => false,
+    ]);
+});
+
+it('deletes old finished runner executions and keeps recent and unfinished ones', function () {
+    $oldFinished = collect([
+        GithubRunnerStatus::Completed,
+        GithubRunnerStatus::Failed,
+        GithubRunnerStatus::Cancelled,
+        GithubRunnerStatus::TimedOut,
+    ])->map(fn (GithubRunnerStatus $status) => cleanupRunnerTestExecution($this->githubApp, $status, 61));
+    $recentFinished = cleanupRunnerTestExecution($this->githubApp, GithubRunnerStatus::Completed, 5);
+    $oldUnfinished = collect(GithubRunnerStatus::active())
+        ->map(fn (GithubRunnerStatus $status) => cleanupRunnerTestExecution($this->githubApp, $status, 61));
+
+    $this->artisan('cleanup:database --yes')
+        ->expectsOutputToContain('Delete 4 entries from github_runner_executions.')
+        ->assertSuccessful();
+
+    expect(GithubRunnerExecution::query()->pluck('id')->sort()->values()->all())
+        ->toBe($oldUnfinished->push($recentFinished)->pluck('id')->sort()->values()->all())
+        ->and($oldFinished)->toHaveCount(4);
+});
+
+it('keeps old finished runner executions in a dry run', function () {
+    cleanupRunnerTestExecution($this->githubApp, GithubRunnerStatus::Completed, 61);
+
+    $this->artisan('cleanup:database')
+        ->expectsOutputToContain('Delete 1 entries from github_runner_executions.')
+        ->assertSuccessful();
+
+    expect(GithubRunnerExecution::query()->count())->toBe(1);
+});
+
+it('prunes expired audit events in batches', function () {
+    $expired = now()->subDays(91);
+    $rows = collect(range(1, 2500))->map(fn (int $i) => [
+        'event' => 'ui.project.updated',
+        'source' => 'ui',
+        'action' => 'updated',
+        'level' => 'info',
+        'actor_type' => 'user',
+        'description' => 'Project updated',
+        'created_at' => $expired,
+    ]);
+    $rows->chunk(500)->each(fn ($chunk) => DB::table('audit_events')->insert($chunk->values()->all()));
+    $recent = AuditEvent::factory()->create(['created_at' => now()->subDays(89)]);
+
+    $deleteQueries = 0;
+    DB::listen(function ($query) use (&$deleteQueries): void {
+        if (str_starts_with(strtolower($query->sql), 'delete from "audit_events"')) {
+            $deleteQueries++;
+        }
+    });
+
+    $deleted = AuditEvent::pruneExpired();
+
+    expect($deleted)->toBe(2500)
+        ->and(AuditEvent::query()->pluck('id')->all())->toBe([$recent->id])
+        ->and($deleteQueries)->toBeGreaterThan(1);
+});