Added mechanics to recover in case of scheduler task that got stuck

server/src/uds/core/jobs/Scheduler.py

@@ -43,7 +43,7 @@ import threading
 import time
 import logging
 
-__updated__ = '2015-10-15'
+__updated__ = '2016-05-18'
 
 logger = logging.getLogger(__name__)
 
@@ -175,6 +175,7 @@ class Scheduler(object):
         logger.debug('Releasing all owned scheduled tasks')
         with transaction.atomic():
             dbScheduler.objects.select_for_update().filter(owner_server=platform.node()).update(owner_server='')  # @UndefinedVariable
+            dbScheduler.objects.select_for_update().filter(last_execution__lt=getSqlDatetime() - timedelta(minutes=15), state=State.RUNNING).update(owner_server='', state=State.FOR_EXECUTE)  # @UndefinedVariable
             dbScheduler.objects.select_for_update().filter(owner_server='').update(state=State.FOR_EXECUTE)  # @UndefinedVariable
 
     def run(self):

server/src/uds/core/workers/SchedulerHouseKeeping.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+#
+# Copyright (c) 2012 Virtual Cable S.L.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''
+@author: Adolfo Gómez, dkmaster at dkmon dot com
+'''
+from __future__ import unicode_literals
+
+from django.db import transaction
+
+from uds.models import Scheduler, getSqlDatetime
+from uds.core.util.State import State
+from uds.core.jobs.Job import Job
+from datetime import timedelta
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SchedulerHousekeeping(Job):
+    frecuency = 301  # Frecuncy for this job
+    friendly_name = 'Scheduler house keeping'
+
+    def __init__(self, environment):
+        super(SchedulerHousekeeping, self).__init__(environment)
+
+    def run(self):
+        '''
+        Look for "hanged" scheduler tasks and reset them
+        '''
+        since = getSqlDatetime() - timedelta(minutes=15)
+        with transaction.atomic():
+            Scheduler.objects.select_for_update().filter(last_execution__lt=since, state=State.RUNNING).update(owner_server='', state=State.FOR_EXECUTE)
+
+