From 207d3f143af2168d042305856735b9aa2af4db98 Mon Sep 17 00:00:00 2001 From: mscherer Date: Wed, 13 May 2026 18:20:56 +0200 Subject: [PATCH 1/2] Cap admin dashboard pending/scheduled details to avoid OOM on backlogs The admin index unconditionally materialised every pending and every scheduled queued_jobs row, then passed both arrays to the view. With a realistic backlog (thousands of unfetched pending jobs) the response balloons in two ways: - the rendered table HTML grows linearly with rows (the modern Bootstrap dashboard is heavier per row than the pre-overhaul one), - DebugKit's Variables panel serialises every view variable and can blow past memory_limit while building its snapshot. Fix: cap the visible pending/scheduled lists at Queue.adminDetailsLimit (default 200), detect truncation with a +1 row probe so we avoid a second count() on small queues, and compute the "new" / pendingJobs / scheduledJobs tile counts via DB count() queries against the same conditions so the tiles still reflect the true totals. Adds QueuedJobsTable::getPendingCount() / getScheduledCount() / getNewCount() to keep the count-query conditions next to the matching SelectQuery getters. Template renders a "Showing N most recent of M" notice when a list was truncated, with a link to the QueuedJobs admin for the full view. Documents Queue.adminDetailsLimit in config/app.example.php. Tests cover both the truncated and within-limit paths. --- config/app.example.php | 9 +++ src/Controller/Admin/QueueController.php | 49 ++++++++++++---- src/Model/Table/QueuedJobsTable.php | 57 +++++++++++++++++++ templates/Admin/Queue/index.php | 37 +++++++++++- .../Controller/Admin/QueueControllerTest.php | 46 +++++++++++++++ 5 files changed, 185 insertions(+), 13 deletions(-) diff --git a/config/app.example.php b/config/app.example.php index 2b37398a..ae54705c 100644 --- a/config/app.example.php +++ b/config/app.example.php @@ -107,6 +107,15 @@ // - string: Uses specified layout 'adminLayout' => null, + // Maximum number of pending and scheduled job rows the admin + // dashboard materialises and renders. Aggregate tile counts on the + // dashboard are still computed via DB count() and reflect the + // unbounded totals; only the visible row list is capped. Raise this + // for more rows at once, lower it if the page is sluggish on a + // large backlog. The cap also keeps DebugKit's Variables panel + // from OOM-ing on huge queues. + 'adminDetailsLimit' => 200, + // Back-to-App link in the admin header (opt-in). When set, an outline // button appears in the top navbar so admins can escape the // plugin-isolated layout. Accepts anything Router::url() takes — Cake diff --git a/src/Controller/Admin/QueueController.php b/src/Controller/Admin/QueueController.php index 8b23beee..56035a10 100644 --- a/src/Controller/Admin/QueueController.php +++ b/src/Controller/Admin/QueueController.php @@ -47,16 +47,31 @@ public function index() { $status = $QueueProcesses->status(); $current = $this->QueuedJobs->getLength(); - $pendingDetails = $this->QueuedJobs->getPendingStats()->toArray(); - $new = 0; - foreach ($pendingDetails as $pendingDetail) { - if ($pendingDetail['fetched'] || $pendingDetail['attempts']) { - continue; - } - $new++; + + // Cap how many pending/scheduled rows we materialise and pass to the + // view. Without a cap a backlog of thousands of pending jobs makes + // the dashboard explode: heavy per-row HTML in the response, and + // DebugKit's Variables panel serialising every entity for its + // snapshot can OOM the request. Aggregate tile counts on the + // dashboard keep using DB-side count() queries so they reflect the + // true totals regardless of the visible-list cap. + $detailsLimit = (int)Configure::read('Queue.adminDetailsLimit', 200); + + // +1 row past the cap so we can detect truncation without a second + // count query in the small-backlog case. + $pendingDetails = $this->QueuedJobs->getPendingStats()->limit($detailsLimit + 1)->toArray(); + $pendingDetailsTruncated = count($pendingDetails) > $detailsLimit; + if ($pendingDetailsTruncated) { + array_pop($pendingDetails); } - $scheduledDetails = $this->QueuedJobs->getScheduledStats()->toArray(); + $new = $this->QueuedJobs->getNewCount(); + + $scheduledDetails = $this->QueuedJobs->getScheduledStats()->limit($detailsLimit + 1)->toArray(); + $scheduledDetailsTruncated = count($scheduledDetails) > $detailsLimit; + if ($scheduledDetailsTruncated) { + array_pop($scheduledDetails); + } $data = $this->QueuedJobs->getStats(); @@ -74,7 +89,17 @@ public function index() { $servers = $QueueProcesses->serverList(); $workers = $status ? $status['workers'] : 0; - $scheduledJobs = count($scheduledDetails); + // True totals from DB. When the visible list is truncated we need + // these for the "showing N of M" hint; when it isn't, they also + // happen to match $pendingDetails count (small extra query that + // avoids one branch in the derivation below). + $totalPending = $pendingDetailsTruncated + ? $this->QueuedJobs->getPendingCount() + : count($pendingDetails); + $scheduledJobs = $scheduledDetailsTruncated + ? $this->QueuedJobs->getScheduledCount() + : count($scheduledDetails); + $runningJobs = $this->QueuedJobs->find() ->where([ 'completed IS' => null, @@ -89,7 +114,7 @@ public function index() { ]) ->count(); // Pending = total pending minus running and failed (to avoid double counting) - $pendingJobs = max(0, count($pendingDetails) - $runningJobs - $failedJobs); + $pendingJobs = max(0, $totalPending - $runningJobs - $failedJobs); $configurations = (array)Configure::read('Queue'); @@ -99,6 +124,10 @@ public function index() { 'data', 'pendingDetails', 'scheduledDetails', + 'pendingDetailsTruncated', + 'scheduledDetailsTruncated', + 'detailsLimit', + 'totalPending', 'status', 'tasks', 'addableTasks', diff --git a/src/Model/Table/QueuedJobsTable.php b/src/Model/Table/QueuedJobsTable.php index aae551a8..0b08119b 100644 --- a/src/Model/Table/QueuedJobsTable.php +++ b/src/Model/Table/QueuedJobsTable.php @@ -1040,6 +1040,46 @@ public function getPendingStats(): SelectQuery { return $this->find('all', ...$findCond); } + /** + * Count of pending jobs (matches getPendingStats() conditions). + * + * Companion to getPendingStats() for callers that LIMIT the materialised + * details list and need the unbounded total to render "showing N of M". + * + * @return int + */ + public function getPendingCount(): int { + return $this->find() + ->where([ + 'completed IS' => null, + 'OR' => [ + 'notbefore <=' => new DateTime(), + 'notbefore IS' => null, + ], + ]) + ->count(); + } + + /** + * Count of pending jobs that have never been fetched or attempted + * (the "new" tile on the admin dashboard). + * + * @return int + */ + public function getNewCount(): int { + return $this->find() + ->where([ + 'completed IS' => null, + 'fetched IS' => null, + 'attempts' => 0, + 'OR' => [ + 'notbefore <=' => new DateTime(), + 'notbefore IS' => null, + ], + ]) + ->count(); + } + /** * @return \Cake\ORM\Query\SelectQuery */ @@ -1068,6 +1108,23 @@ public function getScheduledStats(): SelectQuery { return $this->find('all', ...$findCond); } + /** + * Count of scheduled jobs (matches getScheduledStats() conditions). + * + * Companion to getScheduledStats() for callers that LIMIT the + * materialised details list and need the unbounded total. + * + * @return int + */ + public function getScheduledCount(): int { + return $this->find() + ->where([ + 'completed IS' => null, + 'notbefore >' => new DateTime(), + ]) + ->count(); + } + /** * Cleanup/Delete Completed Jobs. * diff --git a/templates/Admin/Queue/index.php b/templates/Admin/Queue/index.php index 38508aa0..d500c250 100644 --- a/templates/Admin/Queue/index.php +++ b/templates/Admin/Queue/index.php @@ -1,8 +1,12 @@ $taskDescriptions @@ -124,6 +128,23 @@ ) ?>
+ +
+ + Html->link( + __d('queue', 'See QueuedJobs admin'), + ['controller' => 'QueuedJobs', 'action' => 'index'], + ), + ], + ) ?> +
+
@@ -248,9 +269,19 @@
- () + ()
+ +
+ + +
+
diff --git a/tests/TestCase/Controller/Admin/QueueControllerTest.php b/tests/TestCase/Controller/Admin/QueueControllerTest.php index 603eeaa3..9ff026fc 100644 --- a/tests/TestCase/Controller/Admin/QueueControllerTest.php +++ b/tests/TestCase/Controller/Admin/QueueControllerTest.php @@ -71,6 +71,52 @@ public function testIndex() { $this->assertResponseCode(200); } + /** + * The pending/scheduled detail lists must be capped to + * Queue.adminDetailsLimit so the dashboard renders a bounded amount of + * markup and DebugKit's Variables panel doesn't OOM on large backlogs. + * + * @return void + */ + public function testIndexTruncatesPendingDetailsAtDetailsLimit() { + Configure::write('Queue.adminDetailsLimit', 3); + + $QueuedJobs = $this->fetchTable('Queue.QueuedJobs'); + for ($i = 0; $i < 5; $i++) { + $QueuedJobs->createJob('Queue.Example', ['n' => $i]); + } + + $this->get(['prefix' => 'Admin', 'plugin' => 'Queue', 'controller' => 'Queue', 'action' => 'index']); + + $this->assertResponseCode(200); + $pendingDetails = $this->viewVariable('pendingDetails'); + $this->assertCount(3, $pendingDetails); + $this->assertTrue($this->viewVariable('pendingDetailsTruncated')); + $this->assertSame(5, $this->viewVariable('totalPending')); + $this->assertSame(3, $this->viewVariable('detailsLimit')); + } + + /** + * The truncation flag must stay false when the backlog fits inside the + * cap — otherwise the "Showing N of M" hint would render unnecessarily + * and the controller would issue an extra count() it doesn't need. + * + * @return void + */ + public function testIndexNotTruncatedWhenWithinLimit() { + Configure::write('Queue.adminDetailsLimit', 200); + + $QueuedJobs = $this->fetchTable('Queue.QueuedJobs'); + $QueuedJobs->createJob('Queue.Example'); + + $this->get(['prefix' => 'Admin', 'plugin' => 'Queue', 'controller' => 'Queue', 'action' => 'index']); + + $this->assertResponseCode(200); + $this->assertFalse($this->viewVariable('pendingDetailsTruncated')); + $this->assertFalse($this->viewVariable('scheduledDetailsTruncated')); + $this->assertSame(1, $this->viewVariable('totalPending')); + } + /** * @return void */ From 3ba163d3c58817056dab8a3533275cb82981e2cf Mon Sep 17 00:00:00 2001 From: mscherer Date: Wed, 13 May 2026 18:30:14 +0200 Subject: [PATCH 2/2] Pre-filter QueuedJobs admin link from dashboard truncation hints When the dashboard caps the pending/scheduled lists, the 'See QueuedJobs admin' link from the truncation banners now carries ?status=in_progress (pending) or ?status=scheduled, matching the existing search-collection filter names. Lands you exactly on the rows the truncated list represents instead of the full mixed list of completed + failed + running entries. --- templates/Admin/Queue/index.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/templates/Admin/Queue/index.php b/templates/Admin/Queue/index.php index d500c250..dc9ea613 100644 --- a/templates/Admin/Queue/index.php +++ b/templates/Admin/Queue/index.php @@ -139,7 +139,7 @@ $totalPending, $this->Html->link( __d('queue', 'See QueuedJobs admin'), - ['controller' => 'QueuedJobs', 'action' => 'index'], + ['controller' => 'QueuedJobs', 'action' => 'index', '?' => ['status' => 'in_progress']], ), ], ) ?> @@ -277,8 +277,15 @@ Html->link( + __d('queue', 'See QueuedJobs admin'), + ['controller' => 'QueuedJobs', 'action' => 'index', '?' => ['status' => 'scheduled']], + ), + ], ) ?>