request middleware improved. Fixed base model types & migrated providers to 3.x python

server/src/uds/REST/methods/providers.py

@@ -30,8 +30,8 @@
 """
 @author: Adolfo Gómez, dkmaster at dkmon dot com
 """
-import typing
 import logging
+import typing
 
 from django.utils.translation import ugettext, ugettext_lazy as _
 
@@ -78,7 +78,7 @@ class Providers(ModelHandler):
     # Field from where to get "class" and prefix for that class, so this will generate "row-state-A, row-state-X, ....
     table_row_style = {'field': 'maintenance_mode', 'prefix': 'row-maintenance-'}
 
-    def item_as_dict(self, item) -> typing.Dict[str, typing.Any]:
+    def item_as_dict(self, item: Provider) -> typing.Dict[str, typing.Any]:
         type_ = item.getType()
 
         # Icon can have a lot of data (1-2 Kbytes), but it's not expected to have a lot of services providers, and even so, this will work fine
@@ -86,7 +86,8 @@ class Providers(ModelHandler):
             'name': ugettext(t.name()),
             'type': t.type(),
             'description': ugettext(t.description()),
-            'icon': t.icon().replace('\n', '')} for t in type_.getServicesTypes()]
+            'icon': typing.cast(str, t.icon()).replace('\n', '')
+        } for t in type_.getServicesTypes()]
 
         return {
             'id': item.uuid,
@@ -101,22 +102,22 @@ class Providers(ModelHandler):
             'permission': permissions.getEffectivePermission(self._user, item)
         }
 
-    def checkDelete(self, item):
+    def checkDelete(self, item: Provider) -> None:
         if item.services.count() > 0:
-            raise RequestError('Can\'t delete providers with services already associated')
+            raise RequestError(ugettext('Can\'t delete providers with services'))
 
     # Types related
-    def enum_types(self):
+    def enum_types(self) -> typing.Iterable[typing.Type[services.ServiceProvider]]:
         return services.factory().providers().values()
 
     # Gui related
-    def getGui(self, type_):
-        try:
-            return self.addDefaultFields(services.factory().lookup(type_).guiDescription(), ['name', 'comments', 'tags'])
-        except Exception:
-            raise NotFound('type not found')
+    def getGui(self, type_: str) -> typing.List[typing.Any]:
+        clsType = services.factory().lookup(type_)
+        if clsType:
+            return self.addDefaultFields(clsType.guiDescription(), ['name', 'comments', 'tags'])
+        raise NotFound('Type not found!')
 
-    def allservices(self):
+    def allservices(self) -> typing.Generator[typing.Dict, None, None]:
         """
         Custom method that returns "all existing services", no mater who's his daddy :)
         """
@@ -128,7 +129,7 @@ class Providers(ModelHandler):
             except Exception:
                 logger.exception('Passed service cause type is unknown')
 
-    def service(self):
+    def service(self) -> typing.Dict:
         """
         Custom method that returns a service by its uuid, no matter who's his daddy
         """
@@ -139,7 +140,7 @@ class Providers(ModelHandler):
         except Exception:
             raise RequestError(ugettext('Service not found'))
 
-    def maintenance(self, item):
+    def maintenance(self, item: Provider):
         """
         Custom method that swaps maintenance mode state for a provider
         :param item:
@@ -149,12 +150,15 @@ class Providers(ModelHandler):
         item.save()
         return self.item_as_dict(item)
 
-    def test(self, type_):
+    def test(self, type_: str):
         from uds.core.environment import Environment
 
         logger.debug('Type: %s', type_)
         spType = services.factory().lookup(type_)
 
+        if not spType:
+            raise NotFound('Type not found!')
+
         self.ensureAccess(spType, permissions.PERMISSION_MANAGEMENT, root=True)
 
         logger.debug('spType: %s', spType)
@@ -164,5 +168,5 @@ class Providers(ModelHandler):
         res = spType.test(Environment.getTempEnv(), dct)
         if res[0]:
             return 'ok'
-        else:
-            return res[1]
+
+        return res[1]

server/src/uds/REST/model.py

@@ -334,7 +334,7 @@ class DetailHandler(BaseModelHandler):
 
     Also accepts GET methods for "custom" methods
     """
-    custom_methods: typing.ClassVar[typing.Iterable[str]] = []
+    custom_methods: typing.ClassVar[typing.List[str]] = []
     _parent: typing.Optional['ModelHandler']
     _path: str
     _params: typing.Any  # _params is deserialized object from request
@@ -602,15 +602,15 @@ class ModelHandler(BaseModelHandler):
     # This is an array of tuples of two items, where first is method and second inticates if method needs parent id (normal behavior is it needs it)
     # For example ('services', True) -- > .../id_parent/services
     #             ('services', False) --> ..../services
-    custom_methods: typing.ClassVar[typing.Iterable[typing.Tuple[str, bool]]] = []  # If this model respond to "custom" methods, we will declare them here
+    custom_methods: typing.ClassVar[typing.List[typing.Tuple[str, bool]]] = []  # If this model respond to "custom" methods, we will declare them here
     # If this model has details, which ones
     detail: typing.ClassVar[typing.Optional[typing.Dict[str, typing.Type[DetailHandler]]]] = None  # Dictionary containing detail routing
     # Put needed fields
-    save_fields: typing.ClassVar[typing.Iterable[str]] = []
+    save_fields: typing.ClassVar[typing.List[str]] = []
     # Put removable fields before updating
-    remove_fields: typing.ClassVar[typing.Iterable[str]] = []
+    remove_fields: typing.ClassVar[typing.List[str]] = []
     # Table info needed fields and title
-    table_fields: typing.ClassVar[typing.Iterable[typing.Any]] = []
+    table_fields: typing.ClassVar[typing.List[typing.Any]] = []
     table_row_style: typing.ClassVar[typing.Dict] = {}
     table_title: typing.ClassVar[str] = ''
     table_subtitle: typing.ClassVar[str] = ''

server/src/uds/core/util/request.py

@@ -30,8 +30,9 @@
 .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
 """
 import threading
-import logging
 import datetime
+import weakref
+import logging
 import typing
 
 from django.http import HttpRequest, HttpResponse
@@ -41,11 +42,12 @@ from uds.core.util.config import GlobalConfig
 from uds.core.auths.auth import ROOT_ID, USER_KEY, getRootUser
 from uds.models import User
 
-
 logger = logging.getLogger(__name__)
 
-_requests: typing.Dict[int, typing.Tuple[HttpRequest, datetime.datetime]] = {}
+_requests: typing.Dict[int, typing.Tuple[weakref.ref, datetime.datetime]] = {}
 
+# How often to check the requests cache for stuck objects
+CHECK_SECONDS = 3600 * 24  # Once a day is more than enough
 
 def getIdent() -> int:
     ident = threading.current_thread().ident
@@ -55,12 +57,11 @@ def getIdent() -> int:
 def getRequest() -> HttpRequest:
     ident = getIdent()
     if ident in _requests:
-        return _requests[ident][0]
+        return _requests[ident][0]()  # Return obj from weakref
 
     return HttpRequest()
 
 
-
 class GlobalRequestMiddleware:
     lastCheck: typing.ClassVar[datetime.datetime] = datetime.datetime.now()
 
@@ -68,6 +69,9 @@ class GlobalRequestMiddleware:
         self._get_response: typing.Callable[[HttpRequest], HttpResponse] = get_response
 
     def _process_request(self, request: HttpRequest) -> None:
+        # Store request on cache
+        _requests[getIdent()] = (weakref.ref(request), datetime.datetime.now())
+
         # Add IP to request
         GlobalRequestMiddleware.fillIps(request)
         # Ensures request contains os
@@ -75,10 +79,6 @@ class GlobalRequestMiddleware:
         # Ensures that requests contains the valid user
         GlobalRequestMiddleware.getUser(request)
 
-        # Store request on cache
-        _requests[getIdent()] = (request, datetime.datetime.now())
-
-
     def _process_response(self, request: HttpRequest, response: HttpResponse):
         # Remove IP from global cache (processing responses after this will make global request unavailable,
         # but can be got from request again)
@@ -91,6 +91,10 @@ class GlobalRequestMiddleware:
                 logger.info('Request id %s not stored in cache', ident)
         except Exception:
             logger.exception('Deleting stored request')
+
+        # Clean old stored if needed
+        GlobalRequestMiddleware.cleanStuckRequests()
+
         return response
 
     def __call__(self, request: HttpRequest):
@@ -98,17 +102,15 @@ class GlobalRequestMiddleware:
 
         response = self._get_response(request)
 
-        # Clean cache
-        GlobalRequestMiddleware.cleanStuckRequests()
-
         return self._process_response(request, response)
 
     @staticmethod
     def cleanStuckRequests() -> None:
         # In case of some exception, keep clean very old request from time to time...
-        if GlobalRequestMiddleware.lastCheck > datetime.datetime.now() - datetime.timedelta(seconds=60):
+        if GlobalRequestMiddleware.lastCheck > datetime.datetime.now() - datetime.timedelta(seconds=CHECK_SECONDS):
             return
         logger.debug('Cleaning stuck requestws from %s', _requests)
+        # No request lives 60 seconds, so 60 seconds is fine
         cleanFrom: datetime.datetime = datetime.datetime.now() - datetime.timedelta(seconds=60)
         toDelete: typing.List[int] = []
         for ident, request in _requests.items():