Skip to content

Commit de56aa4

Browse files
hoangtrannthomaspaulb
authored andcommitted
[IMP] queue_job: query orphaned dead job not exist in lock table
1 parent 4df7da9 commit de56aa4

File tree

2 files changed

+26
-60
lines changed

2 files changed

+26
-60
lines changed

queue_job/jobrunner/runner.py

Lines changed: 25 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -361,52 +361,26 @@ def _query_requeue_dead_jobs(self):
361361
ELSE exc_info
362362
END)
363363
WHERE
364-
id in (
365-
SELECT
366-
queue_job_id
367-
FROM
368-
queue_job_lock
369-
WHERE
370-
queue_job_id in (
371-
SELECT
372-
id
373-
FROM
374-
queue_job
375-
WHERE
376-
state IN ('enqueued','started')
377-
AND date_enqueued <
378-
(now() AT TIME ZONE 'utc' - INTERVAL '10 sec')
379-
)
380-
FOR UPDATE SKIP LOCKED
381-
)
382-
RETURNING uuid
383-
"""
384-
385-
def _query_requeue_orphaned_jobs(self):
386-
"""Query to requeue jobs stuck in 'enqueued' state without a lock.
387-
388-
This handles the edge case where the runner marks a job as 'enqueued'
389-
but the HTTP request to start the job never reaches the Odoo server
390-
(e.g., due to server shutdown/crash between setting enqueued and
391-
the controller receiving the request). These jobs have no lock record
392-
because set_started() was never called, so they are invisible to
393-
_query_requeue_dead_jobs().
394-
"""
395-
return """
396-
UPDATE
397-
queue_job
398-
SET
399-
state='pending'
400-
WHERE
401-
state = 'enqueued'
364+
state IN ('enqueued','started')
402365
AND date_enqueued < (now() AT TIME ZONE 'utc' - INTERVAL '10 sec')
403-
AND NOT EXISTS (
404-
SELECT
405-
1
406-
FROM
407-
queue_job_lock
408-
WHERE
409-
queue_job_id = queue_job.id
366+
AND (
367+
id in (
368+
SELECT
369+
queue_job_id
370+
FROM
371+
queue_job_lock
372+
WHERE
373+
queue_job_lock.queue_job_id = queue_job.id
374+
FOR UPDATE SKIP LOCKED
375+
)
376+
OR NOT EXISTS (
377+
SELECT
378+
1
379+
FROM
380+
queue_job_lock
381+
WHERE
382+
queue_job_lock.queue_job_id = queue_job.id
383+
)
410384
)
411385
RETURNING uuid
412386
"""
@@ -429,6 +403,12 @@ def requeue_dead_jobs(self):
429403
However, when the Odoo server crashes or is otherwise force-stopped,
430404
running jobs are interrupted while the runner has no chance to know
431405
they have been aborted.
406+
407+
This also handles orphaned jobs (enqueued but never started, no lock).
408+
This edge case occurs when the runner marks a job as 'enqueued'
409+
but the HTTP request to start the job never reaches the Odoo server
410+
(e.g., due to server shutdown/crash between setting enqueued and
411+
the controller receiving the request).
432412
"""
433413

434414
with closing(self.conn.cursor()) as cr:
@@ -439,14 +419,6 @@ def requeue_dead_jobs(self):
439419
for (uuid,) in cr.fetchall():
440420
_logger.warning("Re-queued dead job with uuid: %s", uuid)
441421

442-
# Requeue orphaned jobs (enqueued but never started, no lock)
443-
query = self._query_requeue_orphaned_jobs()
444-
cr.execute(query)
445-
for (uuid,) in cr.fetchall():
446-
_logger.warning(
447-
"Re-queued orphaned job (enqueued without lock) with uuid: %s", uuid
448-
)
449-
450422

451423
class QueueJobRunner:
452424
def __init__(

test_queue_job/tests/test_requeue_dead_job.py

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,14 +93,8 @@ def test_requeue_orphaned_jobs(self):
9393
job_obj.date_enqueued = datetime.now() - timedelta(minutes=1)
9494
job_obj.store()
9595

96-
# job ins't actually picked up by the first requeue attempt
96+
# job is now picked up by the requeue query (which includes orphaned jobs)
9797
query = Database(self.env.cr.dbname)._query_requeue_dead_jobs()
9898
self.env.cr.execute(query)
9999
uuids_requeued = self.env.cr.fetchall()
100-
self.assertFalse(uuids_requeued)
101-
102-
# job is picked up by the 2nd requeue attempt
103-
query = Database(self.env.cr.dbname)._query_requeue_orphaned_jobs()
104-
self.env.cr.execute(query)
105-
uuids_requeued = self.env.cr.fetchall()
106100
self.assertTrue(queue_job.uuid in j[0] for j in uuids_requeued)

0 commit comments

Comments
 (0)