diff --git a/app/Console/Commands/CleanupDatabase.php b/app/Console/Commands/CleanupDatabase.php index eac3a33ebf..eb65007194 100644 --- a/app/Console/Commands/CleanupDatabase.php +++ b/app/Console/Commands/CleanupDatabase.php @@ -2,8 +2,11 @@ namespace App\Console\Commands; +use App\Enums\GithubRunnerStatus; use App\Models\AuditEvent; +use Closure; use Illuminate\Console\Command; +use Illuminate\Database\Query\Builder; use Illuminate\Support\Facades\DB; class CleanupDatabase extends Command @@ -43,22 +46,29 @@ class CleanupDatabase extends Command $this->cleanupTable('application_deployment_queues', 'created_at', now()->subDays($keep_days)); $this->cleanupTable('scheduled_task_executions', 'created_at', now()->subDays($keep_days)); + + $this->cleanupTable('github_runner_executions', 'created_at', now()->subDays($keep_days), fn (Builder $query) => $query + ->whereNotIn('status', array_map(fn (GithubRunnerStatus $status): string => $status->value, GithubRunnerStatus::active()))); } /** * Delete the rows older than the given value in batches, so a large table is not locked * by one long DELETE. + * + * @param (Closure(Builder): mixed)|null $constraint Limits the rows that may be deleted. */ - private function cleanupTable(string $table, string $column, mixed $olderThan): void + private function cleanupTable(string $table, string $column, mixed $olderThan, ?Closure $constraint = null): void { - $count = DB::table($table)->where($column, '<', $olderThan)->count(); + $query = fn (): Builder => DB::table($table)->where($column, '<', $olderThan)->when($constraint, $constraint); + + $count = $query()->count(); $this->line("Delete $count entries from $table."); if (! $this->option('yes')) { return; } do { - $deleted = DB::table($table)->where($column, '<', $olderThan)->limit(self::DELETE_BATCH_SIZE)->delete(); + $deleted = $query()->limit(self::DELETE_BATCH_SIZE)->delete(); } while ($deleted > 0); } } diff --git a/app/Models/AuditEvent.php b/app/Models/AuditEvent.php index b966e55706..2742a3f667 100644 --- a/app/Models/AuditEvent.php +++ b/app/Models/AuditEvent.php @@ -41,6 +41,8 @@ class AuditEvent extends Model 'configuration_snapshot', 'configuration_diff', 'content', 'file_storage_content', ]; + private const PRUNE_BATCH_SIZE = 1000; + protected $fillable = [ 'team_id', 'event', @@ -228,11 +230,23 @@ class AuditEvent extends Model ->first()?->team()?->id; } + /** + * Deletes in batches, so a large table is not locked by one long DELETE. + */ public static function pruneExpired(): int { - return self::query() - ->where('created_at', '<', now()->subDays(90)) - ->delete(); + $olderThan = now()->subDays(90); + $total = 0; + + do { + $deleted = self::query() + ->where('created_at', '<', $olderThan) + ->limit(self::PRUNE_BATCH_SIZE) + ->delete(); + $total += $deleted; + } while ($deleted > 0); + + return $total; } /** diff --git a/tests/Feature/CleanupDatabaseRunnerExecutionsTest.php b/tests/Feature/CleanupDatabaseRunnerExecutionsTest.php new file mode 100644 index 0000000000..2db3949e50 --- /dev/null +++ b/tests/Feature/CleanupDatabaseRunnerExecutionsTest.php @@ -0,0 +1,96 @@ + $githubApp->id, + 'trigger_workflow_job_id' => ++$jobId, + 'status' => $status, + ]); + GithubRunnerExecution::query()->whereKey($execution->id)->toBase()->update(['created_at' => now()->subDays($ageInDays)]); + + return $execution; +} + +beforeEach(function () { + $team = Team::factory()->create(); + $this->githubApp = GithubApp::create([ + 'name' => 'cleanup-runner-app', + 'api_url' => 'https://api.github.com', + 'html_url' => 'https://github.com', + 'custom_user' => 'git', + 'custom_port' => 22, + 'team_id' => $team->id, + 'is_system_wide' => false, + ]); +}); + +it('deletes old finished runner executions and keeps recent and unfinished ones', function () { + $oldFinished = collect([ + GithubRunnerStatus::Completed, + GithubRunnerStatus::Failed, + GithubRunnerStatus::Cancelled, + GithubRunnerStatus::TimedOut, + ])->map(fn (GithubRunnerStatus $status) => cleanupRunnerTestExecution($this->githubApp, $status, 61)); + $recentFinished = cleanupRunnerTestExecution($this->githubApp, GithubRunnerStatus::Completed, 5); + $oldUnfinished = collect(GithubRunnerStatus::active()) + ->map(fn (GithubRunnerStatus $status) => cleanupRunnerTestExecution($this->githubApp, $status, 61)); + + $this->artisan('cleanup:database --yes') + ->expectsOutputToContain('Delete 4 entries from github_runner_executions.') + ->assertSuccessful(); + + expect(GithubRunnerExecution::query()->pluck('id')->sort()->values()->all()) + ->toBe($oldUnfinished->push($recentFinished)->pluck('id')->sort()->values()->all()) + ->and($oldFinished)->toHaveCount(4); +}); + +it('keeps old finished runner executions in a dry run', function () { + cleanupRunnerTestExecution($this->githubApp, GithubRunnerStatus::Completed, 61); + + $this->artisan('cleanup:database') + ->expectsOutputToContain('Delete 1 entries from github_runner_executions.') + ->assertSuccessful(); + + expect(GithubRunnerExecution::query()->count())->toBe(1); +}); + +it('prunes expired audit events in batches', function () { + $expired = now()->subDays(91); + $rows = collect(range(1, 2500))->map(fn (int $i) => [ + 'event' => 'ui.project.updated', + 'source' => 'ui', + 'action' => 'updated', + 'level' => 'info', + 'actor_type' => 'user', + 'description' => 'Project updated', + 'created_at' => $expired, + ]); + $rows->chunk(500)->each(fn ($chunk) => DB::table('audit_events')->insert($chunk->values()->all())); + $recent = AuditEvent::factory()->create(['created_at' => now()->subDays(89)]); + + $deleteQueries = 0; + DB::listen(function ($query) use (&$deleteQueries): void { + if (str_starts_with(strtolower($query->sql), 'delete from "audit_events"')) { + $deleteQueries++; + } + }); + + $deleted = AuditEvent::pruneExpired(); + + expect($deleted)->toBe(2500) + ->and(AuditEvent::query()->pluck('id')->all())->toBe([$recent->id]) + ->and($deleteQueries)->toBeGreaterThan(1); +});