From 06a598d577a857046dee18556b0a547c64979abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20G=C3=B3mez=20Garc=C3=ADa?= Date: Thu, 20 Apr 2023 04:19:57 +0200 Subject: [PATCH] Adding Database logs storage (appart from local files) --- server/src/uds/REST/log.py | 3 +- server/src/uds/__init__.py | 32 ++++++++++-- server/src/uds/auths/SAML/config.py | 2 - server/src/uds/auths/SAML/saml.py | 5 -- server/src/uds/core/managers/log/manager.py | 49 +++++++------------ server/src/uds/core/util/log/__init__.py | 45 ++++++++++++++--- .../src/uds/transports/HTML5SSH/html5ssh.py | 34 ++++++------- 7 files changed, 101 insertions(+), 69 deletions(-) diff --git a/server/src/uds/REST/log.py b/server/src/uds/REST/log.py index fc37992db..22c788817 100644 --- a/server/src/uds/REST/log.py +++ b/server/src/uds/REST/log.py @@ -48,6 +48,7 @@ if typing.TYPE_CHECKING: UUID_REPLACER = ( ('providers', models.Provider), ('services', models.Service), + ('servicespools', models.ServicePool), ('users', models.User), ('groups', models.Group), ) @@ -92,7 +93,7 @@ def logOperation( doLog( None, level=level, - message=f'{handler.request.ip} {username}: [{handler.request.method}/{response_code}] {path}'[ + message=f'{handler.request.ip}[{username}]: [{handler.request.method}/{response_code}] {path}'[ :4096 ], source=LogSource.REST, diff --git a/server/src/uds/__init__.py b/server/src/uds/__init__.py index 965c66229..5b2c39def 100644 --- a/server/src/uds/__init__.py +++ b/server/src/uds/__init__.py @@ -38,6 +38,7 @@ import logging from django.db import connections from django.db.backends.signals import connection_created + # from django.db.models.signals import post_migrate from django.dispatch import receiver @@ -50,8 +51,8 @@ logger = logging.getLogger(__name__) # Set default ssl context unverified, as MOST servers that we will connect will be with self signed certificates... try: - _create_unverified_https_context = ssl._create_unverified_context - ssl._create_default_https_context = _create_unverified_https_context + # _create_unverified_https_context = ssl._create_unverified_context + # ssl._create_default_https_context = _create_unverified_https_context # Capture warnnins to logg logging.captureWarnings(True) @@ -69,17 +70,37 @@ class UDSAppConfig(AppConfig): # with ANY command from manage. logger.debug('Initializing app (ready) ***************') - # Now, ensures that all dynamic elements are loadad and present - # To make sure that the packages are initialized at this point + # Now, ensures that all dynamic elements are loaded and present + # To make sure that the packages are already initialized at this point + + # pylint: disable=unused-import,import-outside-toplevel from . import services + + # pylint: disable=unused-import,import-outside-toplevel from . import auths + + # pylint: disable=unused-import,import-outside-toplevel from . import mfas + + # pylint: disable=unused-import,import-outside-toplevel from . import osmanagers + + # pylint: disable=unused-import,import-outside-toplevel from . import notifiers + + # pylint: disable=unused-import,import-outside-toplevel from . import transports + + # pylint: disable=unused-import,import-outside-toplevel from . import reports + + # pylint: disable=unused-import,import-outside-toplevel from . import dispatchers + + # pylint: disable=unused-import,import-outside-toplevel from . import plugins + + # pylint: disable=unused-import,import-outside-toplevel from . import REST # Ensure notifications table exists on local sqlite db (called "persistent" on settings.py) @@ -96,8 +117,9 @@ default_app_config = 'uds.UDSAppConfig' # Sets up several sqlite non existing methodsm and some optimizations on sqlite +# pylint: disable=unused-argument @receiver(connection_created) -def extend_sqlite(connection=None, **kwargs): +def extend_sqlite(connection=None, **kwargs) -> None: if connection and connection.vendor == "sqlite": logger.debug('Connection vendor is sqlite, extending methods') cursor = connection.cursor() diff --git a/server/src/uds/auths/SAML/config.py b/server/src/uds/auths/SAML/config.py index 415348859..d551af79d 100644 --- a/server/src/uds/auths/SAML/config.py +++ b/server/src/uds/auths/SAML/config.py @@ -4,9 +4,7 @@ from uds.core.util.config import Config ORGANIZATION_NAME = Config.section('SAML').value('Organization Name', 'UDS', help='Organization name to display on SAML SP Metadata') ORGANIZATION_DISPLAY = Config.section('SAML').value('Org. Display Name', 'UDS Organization', help='Organization Display name to display on SAML SP Metadata') ORGANIZATION_URL = Config.section('SAML').value('Organization URL', 'http://www.udsenterprise.com', help='Organization url to display on SAML SP Metadata') -IDP_METADATA_CACHE = Config.section('SAML').value('IDP Metadata cache') ORGANIZATION_NAME.get() ORGANIZATION_DISPLAY.get() ORGANIZATION_URL.get() -IDP_METADATA_CACHE.getInt() diff --git a/server/src/uds/auths/SAML/saml.py b/server/src/uds/auths/SAML/saml.py index aa0a91f75..acf5148d8 100644 --- a/server/src/uds/auths/SAML/saml.py +++ b/server/src/uds/auths/SAML/saml.py @@ -480,11 +480,6 @@ class SAMLAuthenticator(auths.Authenticator): raise auths.exceptions.AuthenticatorException( gettext('Can\'t access idp metadata') ) - self.cache.put( - 'idpMetadata', - val, - config.IDP_METADATA_CACHE.getInt(True), - ) else: val = self.idpMetadata.value diff --git a/server/src/uds/core/managers/log/manager.py b/server/src/uds/core/managers/log/manager.py index 7613cc0ef..02c772e57 100644 --- a/server/src/uds/core/managers/log/manager.py +++ b/server/src/uds/core/managers/log/manager.py @@ -29,8 +29,7 @@ """ @author: Adolfo Gómez, dkmaster at dkmon dot com """ -import traceback -import logging +# import traceback import typing from uds.core.util import singleton @@ -45,9 +44,6 @@ if typing.TYPE_CHECKING: from uds import models -logger = logging.getLogger(__name__) - - class LogManager(metaclass=singleton.Singleton): """ Manager for logging (at database) events @@ -68,12 +64,13 @@ class LogManager(metaclass=singleton.Singleton): message: str, source: str, avoidDuplicates: bool, + logName: str ): """ Logs a message associated to owner """ # Ensure message fits on space - message = str(message)[:255] + message = str(message)[:4096] qs = Log.objects.filter(owner_id=owner_id, owner_type=owner_type.value) # First, ensure we do not have more than requested logs, and we can put one more log item @@ -104,6 +101,7 @@ class LogManager(metaclass=singleton.Singleton): source=source, level=level, data=message, + name=logName, ) except Exception: # nosec # Some objects will not get logged, such as System administrator objects, but this is fine @@ -134,6 +132,7 @@ class LogManager(metaclass=singleton.Singleton): message: str, source: str, avoidDuplicates: bool = True, + logName: typing.Optional[str] = None, ): """ Do the logging for the requested object. @@ -146,25 +145,15 @@ class LogManager(metaclass=singleton.Singleton): else LogObjectType.SYSLOG ) objectId = getattr(wichObject, 'id', -1) + logName = logName or '' if owner_type is not None: try: self._log( - owner_type, objectId, level, message, source, avoidDuplicates + owner_type, objectId, level, message, source, avoidDuplicates, logName ) - except Exception: - logger.error( - 'Error logging: %s:%s %s - %s %s', - owner_type, - objectId, - level, - message, - source, - ) - else: - logger.debug( - 'Requested doLog for a type of object not covered: %s', wichObject - ) + except Exception: # nosec + pass # Can not log, def getLogs( self, wichObject: typing.Optional['Model'], limit: int = -1 @@ -178,7 +167,6 @@ class LogManager(metaclass=singleton.Singleton): if wichObject else LogObjectType.SYSLOG ) - logger.debug('Getting log: %s -> %s', wichObject, owner_type) if owner_type: # 0 is valid owner type return self._getLogs( @@ -187,9 +175,6 @@ class LogManager(metaclass=singleton.Singleton): limit if limit != -1 else owner_type.get_max_elements(), ) - logger.debug( - 'Requested getLogs for a type of object not covered: %s', wichObject - ) return [] def clearLogs(self, wichObject: typing.Optional['Model']): @@ -206,11 +191,11 @@ class LogManager(metaclass=singleton.Singleton): ) if owner_type: self._clearLogs(owner_type, getattr(wichObject, 'id', -1)) - else: - logger.debug( - 'Requested clearLogs for a type of object not covered: %s: %s', - type(wichObject), - wichObject, - ) - for line in traceback.format_stack(limit=5): - logger.debug('>> %s', line) + #else: + # logger.debug( + # 'Requested clearLogs for a type of object not covered: %s: %s', + # type(wichObject), + # wichObject, + #) + #for line in traceback.format_stack(limit=5): + # logger.debug('>> %s', line) diff --git a/server/src/uds/core/util/log/__init__.py b/server/src/uds/core/util/log/__init__.py index 488927008..a1a6a61f2 100644 --- a/server/src/uds/core/util/log/__init__.py +++ b/server/src/uds/core/util/log/__init__.py @@ -30,18 +30,26 @@ """ @author: Adolfo Gómez, dkmaster at dkmon dot com """ +import os import logging import logging.handlers import typing import enum +import re + +from django.apps import apps # Not imported at runtime, just for type checking if typing.TYPE_CHECKING: from django.db.models import Model -logger = logging.getLogger(__name__) useLogger = logging.getLogger('useLog') +# Patter for look for date and time in this format: 2023-04-20 04:03:08,776 +# This is the format used by python logging module +DATETIME_PATTERN: typing.Final[re.Pattern] = re.compile(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})') + + class LogLevel(enum.IntEnum): OTHER = 10000 DEBUG = 20000 @@ -74,6 +82,7 @@ class LogLevel(enum.IntEnum): except ValueError: return cls.OTHER + class LogSource(enum.StrEnum): INTERNAL = 'internal' ACTOR = 'actor' @@ -84,6 +93,8 @@ class LogSource(enum.StrEnum): ADMIN = 'admin' SERVICE = 'service' REST = 'rest' + LOGS = 'logs' + def useLog( type_: str, @@ -124,6 +135,8 @@ def useLog( ) ) + # Will be stored on database by UDSLogHandler + def doLog( wichObject: typing.Optional['Model'], @@ -131,12 +144,12 @@ def doLog( message: str, source: LogSource = LogSource.UNKNOWN, avoidDuplicates: bool = True, + logName: typing.Optional[str] = None, ) -> None: # pylint: disable=import-outside-toplevel from uds.core.managers.log import LogManager - logger.debug('%s %s %s', wichObject, level, message) - LogManager().doLog(wichObject, level, message, source, avoidDuplicates) + LogManager().doLog(wichObject, level, message, source, avoidDuplicates, logName) def getLogs( @@ -166,9 +179,27 @@ class UDSLogHandler(logging.handlers.RotatingFileHandler): Custom log handler that will log to database before calling to RotatingFileHandler """ - def emit(self, record: logging.LogRecord) -> None: - # Currently, simply call to parent - msg = self.format(record) # pylint: disable=unused-variable + # Protects from recursive calls + emiting: typing.ClassVar[bool] = False + + def emit(self, record: logging.LogRecord) -> None: + if apps.ready and record.levelno > logging.INFO and not UDSLogHandler.emiting: + try: + UDSLogHandler.emiting = True + msg = self.format(record) + # Remove date and time from message, as it will be stored on database + msg = DATETIME_PATTERN.sub('', msg) + doLog( + None, + LogLevel.fromInt(record.levelno * 1000), + msg, + LogSource.LOGS, + False, + os.path.basename(self.baseFilename) + ) + except Exception: # nosec: If cannot log, just ignore it + pass + finally: + UDSLogHandler.emiting = False - # TODO: Log message on database and continue as a RotatingFileHandler return super().emit(record) diff --git a/server/src/uds/transports/HTML5SSH/html5ssh.py b/server/src/uds/transports/HTML5SSH/html5ssh.py index 0051268bd..4f39e1c60 100644 --- a/server/src/uds/transports/HTML5SSH/html5ssh.py +++ b/server/src/uds/transports/HTML5SSH/html5ssh.py @@ -79,20 +79,20 @@ class HTML5SSHTransport(transports.Transport): defvalue='https://', length=64, required=True, - tab=gui.TUNNEL_TAB, + tab=gui.Tab.TUNNEL, ) username = gui.TextField( label=_('Username'), order=20, tooltip=_('Username for SSH connection authentication.'), - tab=gui.CREDENTIALS_TAB, + tab=gui.Tab.CREDENTIALS, ) password = gui.PasswordField( label=_('Password'), order=21, tooltip=_('Password for SSH connection authentication'), - tab=gui.CREDENTIALS_TAB, + tab=gui.Tab.CREDENTIALS, ) sshPrivateKey = gui.TextField( label=_('SSH Private Key'), @@ -109,7 +109,7 @@ class HTML5SSHTransport(transports.Transport): tooltip=_( 'Passphrase for SSH private key if it is required. If not provided, but it is needed, user will be prompted for it.' ), - tab=gui.CREDENTIALS_TAB, + tab=gui.Tab.CREDENTIALS, ) sshCommand = gui.TextField( @@ -118,7 +118,7 @@ class HTML5SSHTransport(transports.Transport): tooltip=_( 'Command to execute on the remote server. If not provided, an interactive shell will be executed.' ), - tab=gui.PARAMETERS_TAB, + tab=gui.Tab.PARAMETERS, ) enableFileSharing = gui.ChoiceField( label=_('File Sharing'), @@ -131,7 +131,7 @@ class HTML5SSHTransport(transports.Transport): {'id': 'up', 'text': _('Allow upload only')}, {'id': 'true', 'text': _('Enable file sharing')}, ], - tab=gui.PARAMETERS_TAB, + tab=gui.Tab.PARAMETERS, ) fileSharingRoot = gui.TextField( label=_('File Sharing Root'), @@ -139,7 +139,7 @@ class HTML5SSHTransport(transports.Transport): tooltip=_( 'Root path for file sharing. If not provided, root directory will be used.' ), - tab=gui.PARAMETERS_TAB, + tab=gui.Tab.PARAMETERS, ) sshPort = gui.NumericField( length=40, @@ -148,7 +148,7 @@ class HTML5SSHTransport(transports.Transport): order=33, tooltip=_('Port of the SSH server.'), required=True, - tab=gui.PARAMETERS_TAB, + tab=gui.Tab.PARAMETERS, ) sshHostKey = gui.TextField( label=_('SSH Host Key'), @@ -156,7 +156,7 @@ class HTML5SSHTransport(transports.Transport): tooltip=_( 'Host key of the SSH server. If not provided, no verification of host identity is done.' ), - tab=gui.PARAMETERS_TAB, + tab=gui.Tab.PARAMETERS, ) serverKeepAlive = gui.NumericField( length=3, @@ -168,7 +168,7 @@ class HTML5SSHTransport(transports.Transport): ), required=True, minValue=0, - tab=gui.PARAMETERS_TAB, + tab=gui.Tab.PARAMETERS, ) ticketValidity = gui.NumericField( @@ -181,7 +181,7 @@ class HTML5SSHTransport(transports.Transport): ), required=True, minValue=60, - tab=gui.ADVANCED_TAB, + tab=gui.Tab.ADVANCED, ) forceNewWindow = gui.ChoiceField( order=91, @@ -202,7 +202,7 @@ class HTML5SSHTransport(transports.Transport): ), ], defvalue=gui.FALSE, - tab=gui.ADVANCED_TAB, + tab=gui.Tab.ADVANCED ) def initialize(self, values: 'Module.ValuesType'): @@ -231,13 +231,13 @@ class HTML5SSHTransport(transports.Transport): def getLink( self, - userService: 'models.UserService', + userService: 'models.UserService', # pylint: disable=unused-argument transport: 'models.Transport', ip: str, - os: 'DetectedOsInfo', - user: 'models.User', - password: str, - request: 'ExtendedHttpRequestWithUser', + os: 'DetectedOsInfo', # pylint: disable=unused-argument + user: 'models.User', # pylint: disable=unused-argument + password: str, # pylint: disable=unused-argument + request: 'ExtendedHttpRequestWithUser', # pylint: disable=unused-argument ) -> str: # Build params dict params = {