Added xperimental support for streaming some kind of results.

Currently no one streams anything, way to go to prepare everything for this.
also, removed non-unsed non-maintained filtering

server/src/uds/REST/dispatcher.py

@@ -56,6 +56,7 @@ logger = logging.getLogger(__name__)
 
 __all__ = ['Handler', 'Dispatcher']
 
+
 @dataclasses.dataclass(frozen=True)
 class HandlerNode:
     """
@@ -91,7 +92,9 @@ class Dispatcher(View):
     services: typing.ClassVar[HandlerNode] = HandlerNode('', None, {})
 
     @method_decorator(csrf_exempt)
-    def dispatch(self, request: 'http.request.HttpRequest', *args: typing.Any, **kwargs: typing.Any) -> 'http.HttpResponse':
+    def dispatch(
+        self, request: 'http.request.HttpRequest', *args: typing.Any, **kwargs: typing.Any
+    ) -> 'http.HttpResponse':
         """
         Processes the REST request and routes it wherever it needs to be routed
         """
@@ -183,8 +186,19 @@ class Dispatcher(View):
         try:
             response = operation()
 
-            if not handler.raw:  # Raw handlers will return an HttpResponse Object
-                response = processor.get_response(response)
+            # If response is an HttpResponse object, return it directly
+            if not isinstance(response, http.HttpResponse):
+                # If it is a generator, produce an streamed incremental response
+                if isinstance(response, collections.abc.Generator):
+                    response = typing.cast(
+                        'http.HttpResponse',
+                        http.StreamingHttpResponse(
+                            processor.as_incremental(response),
+                            content_type="application/json",
+                        ),
+                    )
+                else:
+                    response = processor.get_response(response)                
             # Set response headers
             response['UDS-Version'] = f'{consts.system.VERSION};{consts.system.VERSION_STAMP}'
             for k, val in handler.headers().items():
@@ -249,9 +263,7 @@ class Dispatcher(View):
         if name not in service_node.children:
             service_node.children[name] = HandlerNode(name, None, {})
 
-        service_node.children[name] = dataclasses.replace(
-            service_node.children[name], handler=type_
-        )
+        service_node.children[name] = dataclasses.replace(service_node.children[name], handler=type_)
 
     # Initializes the dispatchers
     @staticmethod
@@ -280,4 +292,5 @@ class Dispatcher(View):
             package_name='methods',
         )
 
+
 Dispatcher.initialize()

server/src/uds/REST/handlers.py

@@ -58,7 +58,6 @@ class Handler:
     REST requests handler base class
     """
 
-    raw: typing.ClassVar[bool] = False  # If true, Handler will return directly an HttpResponse Object
     name: typing.ClassVar[typing.Optional[str]] = (
         None  # If name is not used, name will be the class name in lower case
     )

server/src/uds/REST/model/model.py

@@ -31,9 +31,7 @@
 """
 # pylint: disable=too-many-public-methods
 
-import fnmatch
 import logging
-import re
 import typing
 import collections.abc
 
@@ -189,58 +187,6 @@ class ModelHandler(BaseModelHandler):
 
     # End overridable
 
-    def extract_filter(self) -> None:
-        # Extract filter from params if present
-        self.fltr = None
-        if 'filter' in self._params:
-            self.fltr = self._params['filter']
-            del self._params['filter']  # Remove parameter
-            logger.debug('Found a filter expression (%s)', self.fltr)
-
-    def filter(self, data: typing.Any) -> typing.Any:
-        # Right now, filtering only supports a single filter, in a future
-        # we may improve it
-        if self.fltr is None:
-            return data
-
-        # Filtering a non iterable (list or tuple)
-        if not isinstance(data, collections.abc.Iterable):
-            return data
-
-        logger.debug('data: %s, fltr: %s', typing.cast(typing.Any, data), self.fltr)
-        try:
-            fld, pattern = self.fltr.split('=')
-            s, e = '', ''
-            if pattern[0] == '^':
-                pattern = pattern[1:]
-                s = '^'
-            if pattern[-1] == '$':
-                pattern = pattern[:-1]
-                e = '$'
-
-            r = re.compile(s + fnmatch.translate(pattern) + e, re.RegexFlag.IGNORECASE)
-
-            def fltr_function(item: typing.Any) -> bool:
-                if not isinstance(item, dict):
-                    return False
-                try:
-                    if fld not in item or r.match(typing.cast(dict[str, typing.Any], item)[fld]) is None:
-                        return False
-                except Exception:
-                    return False
-                return True
-
-            res: list[dict[str, typing.Any]] = list(
-                filter(fltr_function, typing.cast(collections.abc.Iterable[dict[str, typing.Any]], data))
-            )
-
-            logger.debug('After filtering: %s', res)
-            return res
-        except Exception as e:
-            logger.exception('Exception:')
-            logger.info('Filtering expression %s is invalid!', self.fltr)
-            raise exceptions.rest.RequestError(f'Filtering expression {self.fltr} is invalid') from e
-
     # Helper to process detail
     # Details can be managed (writen) by any user that has MANAGEMENT permission over parent
     def process_detail(self) -> typing.Any:
@@ -338,9 +284,7 @@ class ModelHandler(BaseModelHandler):
         """
         Wraps real get method so we can process filters if they exists
         """
-        # Extract filter from params if present
-        self.extract_filter()
-        return self.filter(self.process_get())
+        return self.process_get()
 
     #  pylint: disable=too-many-return-statements
     def process_get(self) -> typing.Any:

server/src/uds/REST/processors.py

@@ -42,6 +42,8 @@ from django.utils.functional import Promise as DjangoPromise
 
 from uds.core import consts
 
+from .utils import to_incremental_json
+
 # from xml_marshaller import xml_marshaller
 
 logger = logging.getLogger(__name__)
@@ -96,6 +98,12 @@ class ContentProcessor:
         """
         return str(obj)
 
+    def as_incremental(self, obj: typing.Any) -> collections.abc.Iterable[bytes]:
+        """
+        Renders an obj to the specific type, but in an incremental way (if possible)
+        """
+        yield self.render(obj).encode('utf8')
+
     @staticmethod
     def process_for_render(obj: typing.Any) -> typing.Any:
         """
@@ -108,13 +116,19 @@ class ContentProcessor:
             return str(obj)  # This is for translations
 
         if isinstance(obj, dict):
-            return {k: ContentProcessor.process_for_render(v) for k, v in typing.cast(dict[str, typing.Any], obj).items()}
+            return {
+                k: ContentProcessor.process_for_render(v)
+                for k, v in typing.cast(dict[str, typing.Any], obj).items()
+            }
 
         if isinstance(obj, bytes):
             return obj.decode('utf-8')
 
         if isinstance(obj, collections.abc.Iterable):
-            return [ContentProcessor.process_for_render(v) for v in typing.cast(collections.abc.Iterable[typing.Any], obj)]
+            return [
+                ContentProcessor.process_for_render(v)
+                for v in typing.cast(collections.abc.Iterable[typing.Any], obj)
+            ]
 
         if isinstance(obj, (datetime.datetime,)):  # Datetime as timestamp
             return int(time.mktime(obj.timetuple()))
@@ -166,6 +180,10 @@ class JsonProcessor(MarshallerProcessor):
     extensions: typing.ClassVar[collections.abc.Iterable[str]] = ['json']
     marshaller: typing.ClassVar[typing.Any] = json
 
+    def as_incremental(self, obj: typing.Any) -> collections.abc.Iterable[bytes]:
+        for i in to_incremental_json(obj):
+            yield i.encode('utf8')
+
 
 # ---------------
 # XML Processor

server/src/uds/REST/utils.py

@@ -28,8 +28,10 @@
 '''
 Author: Adolfo Gómez, dkmaster at dkmon dot com
 '''
+import json
 import typing
 import re
+import collections.abc
 
 from uds.core.consts.system import VERSION
 from uds.core.util.model import sql_stamp_seconds
@@ -42,6 +44,7 @@ def rest_result(result: typing.Any, **kwargs: typing.Any) -> dict[str, typing.An
     # A common possible value in kwargs is "error"
     return {'result': result, 'stamp': sql_stamp_seconds(), 'version': VERSION, **kwargs}
 
+
 def camel_and_snake_case_from(text: str) -> tuple[str, str]:
     '''
     Returns a tuple with the camel case and snake case of a text
@@ -53,3 +56,20 @@ def camel_and_snake_case_from(text: str) -> tuple[str, str]:
     camel_case_name = camel_case_name[0].lower() + camel_case_name[1:]
 
     return camel_case_name, snake_case_name
+
+
+def to_incremental_json(
+    source: collections.abc.Generator[typing.Any, None, None]
+) -> typing.Generator[str, None, None]:
+    '''
+    Converts a generator to a json incremental string
+    '''
+    yield '['
+    first = True
+    for item in source:
+        if first:
+            first = False
+        else:
+            yield ','
+        yield json.dumps(item)
+    yield ']'