Adding a mechaning to disallow removal of user services by internal method automatically

server/src/uds/core/services/service.py

@@ -269,6 +269,13 @@ class Service(Module):
         Ideally, availability should be cached for a while, so that we don't have to check it every time.
         """
         return True
+    
+    def can_clean_errored_userservices(self) -> bool:
+        """
+        Returns if this service can clean errored services. This is used to check if a service can be cleaned
+        from the stuck cleaner job, for example. By default, this method returns True.
+        """
+        return True
 
     def unmarshal(self, data: bytes) -> None:
         # In fact, we will not unmarshal anything here, but setup maxDeployed

server/src/uds/core/workers/service_pool_cleaner.py

@@ -167,42 +167,42 @@ class DeployedServiceRemover(Job):
 
     def run(self) -> None:
         # First check if there is someone in "removable" estate
-        removableServicePools: collections.abc.Iterable[
+        removable_servicepools: collections.abc.Iterable[
             ServicePool
         ] = ServicePool.objects.filter(state=State.REMOVABLE).order_by('state_date')[
             :10
         ]
 
-        for servicePool in removableServicePools:
+        for servicepool in removable_servicepools:
             try:
                 # Skips checking deployed services in maintenance mode
-                if servicePool.is_in_maintenance() is False:
-                    self.start_removal_of(servicePool)
+                if servicepool.is_in_maintenance() is False:
+                    self.start_removal_of(servicepool)
             except Exception as e1:
-                logger.error('Error removing service pool %s: %s', servicePool.name, e1)
+                logger.error('Error removing service pool %s: %s', servicepool.name, e1)
                 try:
-                    servicePool.delete()
+                    servicepool.delete()
                 except Exception as e2:
                     logger.error('Could not delete %s', e2)
 
-        removingServicePools: collections.abc.Iterable[ServicePool] = ServicePool.objects.filter(
+        already_removing_servicepools: collections.abc.Iterable[ServicePool] = ServicePool.objects.filter(
             state=State.REMOVING
         ).order_by('state_date')[:10]
         # Check if they have been removing for a long time.
         # Note. if year is 1972, it comes from a previous version, set state_date to now
         # If in time and not in maintenance mode, continue removing
-        for servicePool in removingServicePools:
+        for servicepool in already_removing_servicepools:
             try:
-                if servicePool.state_date.year == 1972:
-                    servicePool.state_date = sql_datetime()
-                    servicePool.save(update_fields=['state_date'])
-                if servicePool.state_date < sql_datetime() - timedelta(
+                if servicepool.state_date.year == 1972:
+                    servicepool.state_date = sql_datetime()
+                    servicepool.save(update_fields=['state_date'])
+                if servicepool.state_date < sql_datetime() - timedelta(
                     seconds=MAX_REMOVING_TIME
                 ):
-                    self.force_removal_of(servicePool)  # Force removal
+                    self.force_removal_of(servicepool)  # Force removal
 
                 # Skips checking deployed services in maintenance mode
-                if servicePool.is_in_maintenance() is False:
-                    self.continue_removal_of(servicePool)
+                if servicepool.is_in_maintenance() is False:
+                    self.continue_removal_of(servicepool)
             except Exception:
                 logger.exception('Removing deployed service')

server/src/uds/core/workers/stuck_cleaner.py

@@ -43,9 +43,7 @@ from uds.core.util import log
 
 logger = logging.getLogger(__name__)
 
-MAX_STUCK_TIME = (
-    3600 * 24
-)  # At most 1 days "Stuck", not configurable (there is no need to)
+MAX_STUCK_TIME = 3600 * 24  # At most 1 days "Stuck", not configurable (there is no need to)
 
 
 class StuckCleaner(Job):
@@ -70,10 +68,7 @@ class StuckCleaner(Job):
                         Q(
                             userServices__state=State.PREPARING,
                         )
-                        | ~Q(
-                            userServices__state__in=State.INFO_STATES
-                            + State.VALID_STATES
-                        )
+                        | ~Q(userServices__state__in=State.INFO_STATES + State.VALID_STATES)
                     ),
                 )
             )
@@ -89,12 +84,14 @@ class StuckCleaner(Job):
             yield from q.exclude(state__in=State.INFO_STATES + State.VALID_STATES)
             yield from q.filter(state=State.PREPARING)
 
-        for servicePool in servicePoolswithStucks:
+        for servicepool in servicePoolswithStucks:
+            if servicepool.service.get_instance().can_clean_errored_userservices() is False:
+                continue
             # logger.debug('Searching for stuck states for %s', servicePool.name)
-            for stuck in _retrieve_stuck_user_services(servicePool):
+            for stuck in _retrieve_stuck_user_services(servicepool):
                 logger.debug('Found stuck user service %s', stuck)
                 log.log(
-                    servicePool,
+                    servicepool,
                     log.LogLevel.ERROR,
                     f'User service {stuck.name} has been hard removed because it\'s stuck',
                 )