Enhance logging capabilities and refactor matcher function usage

- Added 'CLIENT' as a log source in LogSource enum.
- Updated translations to include "Logs".
- Refactored matcher function to match for consistency across multiple files.
- Modified log method in UserService and User classes to accept a source parameter.
- Adjusted LogMaintenance job frequency from two hours to one hour and added a condition for log removal based on creation time.

server/src/uds/REST/methods/client.py

@@ -34,12 +34,14 @@ import typing
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
+from uds import models
 from uds.core import consts, exceptions, types
 from uds.core.managers.crypto import CryptoManager
 from uds.core.managers.userservice import UserServiceManager
 from uds.core.exceptions.services import ServiceNotReadyError
+from uds.core.types.log import LogLevel, LogSource
 from uds.core.util.config import GlobalConfig
-from uds.core.util.rest.tools import matcher
+from uds.core.util.rest.tools import match
 from uds.models import TicketStore, User
 from uds.REST import Handler
 
@@ -159,13 +161,25 @@ class Client(Handler):
             )
 
             logger.debug('Script: %s', transport_script)
-            
-            _log_ticket = TicketStore.create(
-                {
-                    'user': self._request.user.uuid,
-                    'type': 'log',
-                }
-            )
+
+            # is_logging_enabled = self._request.user.properties.get('log_enabled', False)
+            is_logging_enabled = True
+            log: dict[str, 'str|None'] = {
+                'level': 'DEBUG',
+                'ticket': None,
+            }
+
+            if is_logging_enabled:
+                log['ticket'] = TicketStore.create(
+                    {
+                        'user': self._request.user.uuid,
+                        'userservice': info.userservice.uuid,
+                        'type': 'log',
+                    },
+                    # Long enough for a looong time, will be cleaned on first access
+                    # Or 24 hours after creation, whatever happens first
+                    validity=60 * 60 * 24,
+                )
 
             return Client.result(
                 result={
@@ -173,10 +187,7 @@ class Client(Handler):
                     'type': transport_script.script_type,
                     'signature': transport_script.signature_b64,  # It is already on base64
                     'params': transport_script.encoded_parameters,
-                    # 'log': {
-                    #     'level': 'DEBUG',
-                    #     'ticket': _log_ticket,
-                    # }
+                    'log': log,
                 }
             )
         except ServiceNotReadyError as e:
@@ -201,7 +212,7 @@ class Client(Handler):
 
         Currently, only "upload logs"
         """
-        logger.debug('Client args for PUT: %s', self._args)
+        logger.debug('Client args for POST: %s', self._args)
         try:
             ticket, command = self._args[:2]
             try:
@@ -211,14 +222,27 @@ class Client(Handler):
 
             self._request.user = User.objects.get(uuid=data['user'])
 
+            try:
+                userservice = models.UserService.objects.get(uuid=data['userservice'])
+            except models.UserService.DoesNotExist:
+                return Client.result(error='Service not found')
+
             match command:
                 case 'log':
                     if data.get('type') != 'log':
                         return Client.result(error='Invalid command')
 
-                    log = self._params.get('log', '')
+                    log: str = self._params.get('log', '')
                     # Right now, log to logger, but will be stored with user logs
-                    logger.info('Client log for %s: %s', self._request.user.pretty_name, log)
+                    for line in log.split('\n'):
+                        # Firt word is level
+                        try:
+                            level, message = line.split(' ', 1)
+                            userservice.log(message, LogLevel.from_str(level), LogSource.CLIENT)
+                        except Exception:
+                            # If something goes wrong, log it as debug
+                            pass
+                        # logger.info('Client log for %s: %s', self._request.user.pretty_name, line)
                 case _:
                     return Client.result(error='Invalid command')
 
@@ -250,7 +274,7 @@ class Client(Handler):
                 }
             )
 
-        return matcher(
+        return match(
             self._args,
             _error,  # In case of error, raises RequestError
             ((), _noargs),  # No args, return version

server/src/uds/REST/methods/connection.py

@@ -38,7 +38,7 @@ from uds.core import exceptions, types
 from uds.core.managers.crypto import CryptoManager
 from uds.core.managers.userservice import UserServiceManager
 from uds.core.exceptions.services import ServiceNotReadyError
-from uds.core.util.rest.tools import matcher
+from uds.core.util.rest.tools import match
 from uds.REST import Handler
 from uds.web.util import services
 
@@ -179,7 +179,7 @@ class Connection(Handler):
         def error() -> dict[str, typing.Any]:
             raise exceptions.rest.RequestError('Invalid Request')
 
-        return matcher(
+        return match(
             self._args,
             error,
             ((), self.service_list),

server/src/uds/REST/methods/permissions.py

@@ -38,7 +38,7 @@ import uds.core.types.permissions
 from uds import models
 from uds.core import exceptions
 from uds.core.util import permissions
-from uds.core.util.rest.tools import matcher
+from uds.core.util.rest.tools import match
 from uds.REST import Handler
 
 # Not imported at runtime, just for type checking
@@ -155,7 +155,7 @@ class Permissions(Handler):
             raise exceptions.rest.RequestError('Invalid request')
 
         # match is a helper function that will match the args with the given patterns
-        return matcher(self._args,
+        return match(self._args,
             no_match,
             (('<cls>', '<obj>', 'users', 'add', '<user>'), add_user_permission),
             (('<cls>', '<obj>', 'groups', 'add', '<group>'), add_group_permission),

server/src/uds/REST/methods/reports.py

@@ -36,7 +36,7 @@ import typing
 from django.utils.translation import gettext_lazy as _
 
 from uds.core import types, consts
-from uds.core.util.rest.tools import matcher
+from uds.core.util.rest.tools import match
 from uds.REST import model
 from uds import reports
 
@@ -98,7 +98,7 @@ class Reports(model.BaseModelHandler):
         def report_gui(report_id: str) -> typing.Any:
             return self.get_gui(report_id)
 
-        return matcher(
+        return match(
             self._args,
             error,
             ((), lambda: list(self.get_items())),

server/src/uds/core/types/log.py

@@ -86,6 +86,7 @@ class LogSource(enum.StrEnum):
     REST = 'rest'
     LOGS = 'logs'
     MODULE = 'module'
+    CLIENT = 'client'
 
 
 # Note: Once assigned a value, do not change it, as it will break the log

server/src/uds/core/util/rest/tools.py

@@ -48,7 +48,7 @@ T = typing.TypeVar('T', bound=typing.Any)
 # The callback will be called with the arguments in the order they are in the tuple, so:
 # callback(sample, arg_2, argument)
 # And the literals will be ignored
-def matcher(
+def match(
     arg_list: collections.abc.Iterable[str],
     error: collections.abc.Callable[..., typing.Any],
     *args: tuple[tuple[str, ...], collections.abc.Callable[..., T]],

server/src/uds/models/user.py

@@ -118,6 +118,15 @@ class User(UUIDModel, properties.PropertiesMixin):
         """
         return self.manager.get_instance()
 
+    # Utility for logging
+    def log(
+        self,
+        message: str,
+        level: types.log.LogLevel = types.log.LogLevel.INFO,
+        source: types.log.LogSource = types.log.LogSource.INTERNAL,
+    ) -> None:
+        log.log(self, level, message, source)
+
     def is_staff(self) -> bool:
         """
         Return true if this user is admin or staff member
@@ -164,7 +173,9 @@ class User(UUIDModel, properties.PropertiesMixin):
                 number_belongs_meta=Count('groups', filter=Q(groups__id__in=grps))
             )  # g.groups.filter(id__in=grps).count()
         ):
-            number_of_groups_belonging_in_meta: int = typing.cast(typing.Any, g).number_belongs_meta  # Anotated field
+            number_of_groups_belonging_in_meta: int = typing.cast(
+                typing.Any, g
+            ).number_belongs_meta  # Anotated field
 
             logger.debug('gn = %s', number_of_groups_belonging_in_meta)
             logger.debug('groups count: %s', typing.cast(typing.Any, g).number_groups)  # Anotated field
@@ -212,9 +223,7 @@ class User(UUIDModel, properties.PropertiesMixin):
         # Remove related stored values
         try:
             storage.StorageAsDict(
-                owner='manager' + str(to_delete.manager.uuid),
-                group=None,
-                atomic=False
+                owner='manager' + str(to_delete.manager.uuid), group=None, atomic=False
             ).clear()
         except Exception:
             logger.exception('Removing stored data')

server/src/uds/models/user_service.py

@@ -639,8 +639,8 @@ class UserService(UUIDModel, properties.PropertiesMixin):
         )
 
     # Utility for logging
-    def log(self, message: str, level: types.log.LogLevel = types.log.LogLevel.INFO) -> None:
-        log.log(self, level, message, types.log.LogSource.INTERNAL)
+    def log(self, message: str, level: types.log.LogLevel = types.log.LogLevel.INFO, source: types.log.LogSource = types.log.LogSource.INTERNAL) -> None:
+        log.log(self, level, message, source)
 
     def test_connectivity(self, host: str, port: 'str|int', timeout: int = 4) -> bool:
         return self.deployed_service.test_connectivity(host, port, timeout)

server/src/uds/static/admin/3rdpartylicenses.txt

@@ -246,6 +246,27 @@ See `/licenses/LICENSE-d3` for details of the license.
 Package: @angular/core
 License: "MIT"
 
+The MIT License
+
+Copyright (c) 2010-2024 Google LLC. https://angular.dev/license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Package: rxjs
@@ -474,16 +495,79 @@ PERFORMANCE OF THIS SOFTWARE.
 Package: @angular/common
 License: "MIT"
 
+The MIT License
+
+Copyright (c) 2010-2024 Google LLC. https://angular.dev/license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Package: @angular/platform-browser
 License: "MIT"
 
+The MIT License
+
+Copyright (c) 2010-2024 Google LLC. https://angular.dev/license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Package: @angular/router
 License: "MIT"
 
+The MIT License
+
+Copyright (c) 2010-2024 Google LLC. https://angular.dev/license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Package: @angular/cdk
@@ -541,11 +625,53 @@ THE SOFTWARE.
 Package: @angular/animations
 License: "MIT"
 
+The MIT License
+
+Copyright (c) 2010-2024 Google LLC. https://angular.dev/license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Package: @angular/forms
 License: "MIT"
 
+The MIT License
+
+Copyright (c) 2010-2024 Google LLC. https://angular.dev/license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Package: ngx-echarts

server/src/uds/static/admin/translations-fakejs.js

@@ -431,6 +431,7 @@ gettext("Information for");
 gettext("Groups");
 gettext("Services Pools");
 gettext("Assigned Services");
+gettext("Logs");
 gettext("Ok");
 gettext("Summary");
 gettext("Users");

server/src/uds/workers/log.py

@@ -29,6 +29,7 @@
 """
 Author: Adolfo Gómez, dkmaster at dkmon dot com
 """
+import datetime
 import logging
 
 from django.db.models import Count
@@ -36,6 +37,7 @@ from django.db.models import Count
 from uds.core.jobs import Job
 from uds import models
 from uds.core.types import log
+from uds.core.util.model import sql_now
 
 # from uds.core.util.config import GlobalConfig
 
@@ -44,7 +46,7 @@ logger = logging.getLogger(__name__)
 
 
 class LogMaintenance(Job):
-    frecuency = 7200  # Once every two hours
+    frecuency = 3600  # Once every hour
     # frecuency_cfg = GlobalConfig.XXXX
     friendly_name = 'Log maintenance'
 
@@ -65,9 +67,14 @@ class LogMaintenance(Job):
                 continue
 
             max_elements = owner_type.get_max_elements()
+            removing_before = sql_now() - datetime.timedelta(seconds=3600)
             if 0 < max_elements < count:  # Negative max elements means "unlimited"
                 # We will delete the oldest ones
-                for record in models.Log.objects.filter(owner_id=owner_id, owner_type=owner_type).order_by(
+                for record in models.Log.objects.filter(
+                    owner_id=owner_id,
+                    owner_type=owner_type,
+                    created_lt=removing_before,
+                ).order_by(
                     'created', 'id'
                 )[: count - max_elements + 1]:
                     record.delete()