burmatovan/openuds
1
0
Fork 0
forked from shaba/openuds
Code Issues Pull Requests Projects Releases Wiki Activity

Retry scheduler cleanup in case of locked db

Browse Source
This commit is contained in:
Adolfo Gómez García 2021-01-29 11:54:48 +01:00
parent 5b8ff497fa
commit d2d190e8a4
1 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
server/src/uds/core/workers/scheduler_housekeeping.py
View File

@ -30,6 +30,7 @@
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
from datetime import timedelta
import time
import logging
from django.db import transaction
@ -56,7 +57,14 @@ class SchedulerHousekeeping(Job):
Look for "hanged" scheduler tasks and reschedule them
"""
since = getSqlDatetime() - timedelta(minutes=MAX_EXECUTION_MINUTES)
with transaction.atomic():
Scheduler.objects.select_for_update().filter(
last_execution__lt=since, state=State.RUNNING
).update(owner_server='', state=State.FOR_EXECUTE)
for i in range(3): # Retry three times in case of lockout error
try:
with transaction.atomic():
Scheduler.objects.select_for_update(skip_locked=True).filter(
last_execution__lt=since, state=State.RUNNING
).update(owner_server='', state=State.FOR_EXECUTE)
break
except Exception as e:
logger.info('Retrying Scheduler cleanup transaction')
time.sleep(1)