shaba/openuds
1
0
Fork 1
mirror of https://github.com/dkmstr/openuds.git synced 2025-01-10 01:17:59 +03:00
Code Issues Releases Wiki Activity

Adding Database logs storage (appart from local files)

Browse Source
This commit is contained in:
Adolfo Gómez García 2023-04-20 04:19:57 +02:00
parent 9030ff3ab3
commit 06a598d577
No known key found for this signature in database
GPG Key ID: DD1ABF20724CDA23
7 changed files with 101 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/uds/REST/log.py
View File

@ -48,6 +48,7 @@ if typing.TYPE_CHECKING:
UUID_REPLACER = ( UUID_REPLACER = (
('providers', models.Provider), ('providers', models.Provider),
('services', models.Service), ('services', models.Service),
('servicespools', models.ServicePool),
('users', models.User), ('users', models.User),
('groups', models.Group), ('groups', models.Group),
) )
@ -92,7 +93,7 @@ def logOperation(
doLog( doLog(
None, None,
level=level, level=level,
message=f'{handler.request.ip} {username}: [{handler.request.method}/{response_code}] {path}'[ message=f'{handler.request.ip}[{username}]: [{handler.request.method}/{response_code}] {path}'[
:4096 :4096
], ],
source=LogSource.REST, source=LogSource.REST,

32
server/src/uds/__init__.py
View File

@ -38,6 +38,7 @@ import logging
from django.db import connections from django.db import connections
from django.db.backends.signals import connection_created from django.db.backends.signals import connection_created
# from django.db.models.signals import post_migrate # from django.db.models.signals import post_migrate
from django.dispatch import receiver from django.dispatch import receiver
@ -50,8 +51,8 @@ logger = logging.getLogger(__name__)
# Set default ssl context unverified, as MOST servers that we will connect will be with self signed certificates... # Set default ssl context unverified, as MOST servers that we will connect will be with self signed certificates...
try: try:
_create_unverified_https_context = ssl._create_unverified_context # _create_unverified_https_context = ssl._create_unverified_context
ssl._create_default_https_context = _create_unverified_https_context # ssl._create_default_https_context = _create_unverified_https_context
# Capture warnnins to logg # Capture warnnins to logg
logging.captureWarnings(True) logging.captureWarnings(True)
@ -69,17 +70,37 @@ class UDSAppConfig(AppConfig):
# with ANY command from manage. # with ANY command from manage.
logger.debug('Initializing app (ready) ***************') logger.debug('Initializing app (ready) ***************')
# Now, ensures that all dynamic elements are loadad and present # Now, ensures that all dynamic elements are loaded and present
# To make sure that the packages are initialized at this point # To make sure that the packages are already initialized at this point
# pylint: disable=unused-import,import-outside-toplevel
from . import services from . import services
# pylint: disable=unused-import,import-outside-toplevel
from . import auths from . import auths
# pylint: disable=unused-import,import-outside-toplevel
from . import mfas from . import mfas
# pylint: disable=unused-import,import-outside-toplevel
from . import osmanagers from . import osmanagers
# pylint: disable=unused-import,import-outside-toplevel
from . import notifiers from . import notifiers
# pylint: disable=unused-import,import-outside-toplevel
from . import transports from . import transports
# pylint: disable=unused-import,import-outside-toplevel
from . import reports from . import reports
# pylint: disable=unused-import,import-outside-toplevel
from . import dispatchers from . import dispatchers
# pylint: disable=unused-import,import-outside-toplevel
from . import plugins from . import plugins
# pylint: disable=unused-import,import-outside-toplevel
from . import REST from . import REST
# Ensure notifications table exists on local sqlite db (called "persistent" on settings.py) # Ensure notifications table exists on local sqlite db (called "persistent" on settings.py)
@ -96,8 +117,9 @@ default_app_config = 'uds.UDSAppConfig'
# Sets up several sqlite non existing methodsm and some optimizations on sqlite # Sets up several sqlite non existing methodsm and some optimizations on sqlite
# pylint: disable=unused-argument
@receiver(connection_created) @receiver(connection_created)
def extend_sqlite(connection=None, **kwargs): def extend_sqlite(connection=None, **kwargs) -> None:
if connection and connection.vendor == "sqlite": if connection and connection.vendor == "sqlite":
logger.debug('Connection vendor is sqlite, extending methods') logger.debug('Connection vendor is sqlite, extending methods')
cursor = connection.cursor() cursor = connection.cursor()

2
server/src/uds/auths/SAML/config.py
View File

@ -4,9 +4,7 @@ from uds.core.util.config import Config
ORGANIZATION_NAME = Config.section('SAML').value('Organization Name', 'UDS', help='Organization name to display on SAML SP Metadata') ORGANIZATION_NAME = Config.section('SAML').value('Organization Name', 'UDS', help='Organization name to display on SAML SP Metadata')
ORGANIZATION_DISPLAY = Config.section('SAML').value('Org. Display Name', 'UDS Organization', help='Organization Display name to display on SAML SP Metadata') ORGANIZATION_DISPLAY = Config.section('SAML').value('Org. Display Name', 'UDS Organization', help='Organization Display name to display on SAML SP Metadata')
ORGANIZATION_URL = Config.section('SAML').value('Organization URL', 'http://www.udsenterprise.com', help='Organization url to display on SAML SP Metadata') ORGANIZATION_URL = Config.section('SAML').value('Organization URL', 'http://www.udsenterprise.com', help='Organization url to display on SAML SP Metadata')
IDP_METADATA_CACHE = Config.section('SAML').value('IDP Metadata cache')
ORGANIZATION_NAME.get() ORGANIZATION_NAME.get()
ORGANIZATION_DISPLAY.get() ORGANIZATION_DISPLAY.get()
ORGANIZATION_URL.get() ORGANIZATION_URL.get()
IDP_METADATA_CACHE.getInt()

5
server/src/uds/auths/SAML/saml.py
View File

@ -480,11 +480,6 @@ class SAMLAuthenticator(auths.Authenticator):
raise auths.exceptions.AuthenticatorException( raise auths.exceptions.AuthenticatorException(
gettext('Can\'t access idp metadata') gettext('Can\'t access idp metadata')
) )
self.cache.put(
'idpMetadata',
val,
config.IDP_METADATA_CACHE.getInt(True),
)
else: else:
val = self.idpMetadata.value val = self.idpMetadata.value

49
server/src/uds/core/managers/log/manager.py
View File

@ -29,8 +29,7 @@
""" """
@author: Adolfo Gómez, dkmaster at dkmon dot com @author: Adolfo Gómez, dkmaster at dkmon dot com
""" """
import traceback # import traceback
import logging
import typing import typing
from uds.core.util import singleton from uds.core.util import singleton
@ -45,9 +44,6 @@ if typing.TYPE_CHECKING:
from uds import models from uds import models
logger = logging.getLogger(__name__)
class LogManager(metaclass=singleton.Singleton): class LogManager(metaclass=singleton.Singleton):
""" """
Manager for logging (at database) events Manager for logging (at database) events
@ -68,12 +64,13 @@ class LogManager(metaclass=singleton.Singleton):
message: str, message: str,
source: str, source: str,
avoidDuplicates: bool, avoidDuplicates: bool,
logName: str
): ):
""" """
Logs a message associated to owner Logs a message associated to owner
""" """
# Ensure message fits on space # Ensure message fits on space
message = str(message)[:255] message = str(message)[:4096]
qs = Log.objects.filter(owner_id=owner_id, owner_type=owner_type.value) qs = Log.objects.filter(owner_id=owner_id, owner_type=owner_type.value)
# First, ensure we do not have more than requested logs, and we can put one more log item # First, ensure we do not have more than requested logs, and we can put one more log item
@ -104,6 +101,7 @@ class LogManager(metaclass=singleton.Singleton):
source=source, source=source,
level=level, level=level,
data=message, data=message,
name=logName,
) )
except Exception: # nosec except Exception: # nosec
# Some objects will not get logged, such as System administrator objects, but this is fine # Some objects will not get logged, such as System administrator objects, but this is fine
@ -134,6 +132,7 @@ class LogManager(metaclass=singleton.Singleton):
message: str, message: str,
source: str, source: str,
avoidDuplicates: bool = True, avoidDuplicates: bool = True,
logName: typing.Optional[str] = None,
): ):
""" """
Do the logging for the requested object. Do the logging for the requested object.
@ -146,25 +145,15 @@ class LogManager(metaclass=singleton.Singleton):
else LogObjectType.SYSLOG else LogObjectType.SYSLOG
) )
objectId = getattr(wichObject, 'id', -1) objectId = getattr(wichObject, 'id', -1)
logName = logName or ''
if owner_type is not None: if owner_type is not None:
try: try:
self._log( self._log(
owner_type, objectId, level, message, source, avoidDuplicates owner_type, objectId, level, message, source, avoidDuplicates, logName
)
except Exception:
logger.error(
'Error logging: %s:%s %s - %s %s',
owner_type,
objectId,
level,
message,
source,
)
else:
logger.debug(
'Requested doLog for a type of object not covered: %s', wichObject
) )
except Exception: # nosec
pass # Can not log,
def getLogs( def getLogs(
self, wichObject: typing.Optional['Model'], limit: int = -1 self, wichObject: typing.Optional['Model'], limit: int = -1
@ -178,7 +167,6 @@ class LogManager(metaclass=singleton.Singleton):
if wichObject if wichObject
else LogObjectType.SYSLOG else LogObjectType.SYSLOG
) )
logger.debug('Getting log: %s -> %s', wichObject, owner_type)
if owner_type: # 0 is valid owner type if owner_type: # 0 is valid owner type
return self._getLogs( return self._getLogs(
@ -187,9 +175,6 @@ class LogManager(metaclass=singleton.Singleton):
limit if limit != -1 else owner_type.get_max_elements(), limit if limit != -1 else owner_type.get_max_elements(),
) )
logger.debug(
'Requested getLogs for a type of object not covered: %s', wichObject
)
return [] return []
def clearLogs(self, wichObject: typing.Optional['Model']): def clearLogs(self, wichObject: typing.Optional['Model']):
@ -206,11 +191,11 @@ class LogManager(metaclass=singleton.Singleton):
) )
if owner_type: if owner_type:
self._clearLogs(owner_type, getattr(wichObject, 'id', -1)) self._clearLogs(owner_type, getattr(wichObject, 'id', -1))
else: #else:
logger.debug( # logger.debug(
'Requested clearLogs for a type of object not covered: %s: %s', # 'Requested clearLogs for a type of object not covered: %s: %s',
type(wichObject), # type(wichObject),
wichObject, # wichObject,
) #)
for line in traceback.format_stack(limit=5): #for line in traceback.format_stack(limit=5):
logger.debug('>> %s', line) # logger.debug('>> %s', line)

45
server/src/uds/core/util/log/__init__.py
View File

@ -30,18 +30,26 @@
""" """
@author: Adolfo Gómez, dkmaster at dkmon dot com @author: Adolfo Gómez, dkmaster at dkmon dot com
""" """
import os
import logging import logging
import logging.handlers import logging.handlers
import typing import typing
import enum import enum
import re
from django.apps import apps
# Not imported at runtime, just for type checking # Not imported at runtime, just for type checking
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from django.db.models import Model from django.db.models import Model
logger = logging.getLogger(__name__)
useLogger = logging.getLogger('useLog') useLogger = logging.getLogger('useLog')
# Patter for look for date and time in this format: 2023-04-20 04:03:08,776
# This is the format used by python logging module
DATETIME_PATTERN: typing.Final[re.Pattern] = re.compile(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})')
class LogLevel(enum.IntEnum): class LogLevel(enum.IntEnum):
OTHER = 10000 OTHER = 10000
DEBUG = 20000 DEBUG = 20000
@ -74,6 +82,7 @@ class LogLevel(enum.IntEnum):
except ValueError: except ValueError:
return cls.OTHER return cls.OTHER
class LogSource(enum.StrEnum): class LogSource(enum.StrEnum):
INTERNAL = 'internal' INTERNAL = 'internal'
ACTOR = 'actor' ACTOR = 'actor'
@ -84,6 +93,8 @@ class LogSource(enum.StrEnum):
ADMIN = 'admin' ADMIN = 'admin'
SERVICE = 'service' SERVICE = 'service'
REST = 'rest' REST = 'rest'
LOGS = 'logs'
def useLog( def useLog(
type_: str, type_: str,
@ -124,6 +135,8 @@ def useLog(
) )
) )
# Will be stored on database by UDSLogHandler
def doLog( def doLog(
wichObject: typing.Optional['Model'], wichObject: typing.Optional['Model'],
@ -131,12 +144,12 @@ def doLog(
message: str, message: str,
source: LogSource = LogSource.UNKNOWN, source: LogSource = LogSource.UNKNOWN,
avoidDuplicates: bool = True, avoidDuplicates: bool = True,
logName: typing.Optional[str] = None,
) -> None: ) -> None:
# pylint: disable=import-outside-toplevel # pylint: disable=import-outside-toplevel
from uds.core.managers.log import LogManager from uds.core.managers.log import LogManager
logger.debug('%s %s %s', wichObject, level, message) LogManager().doLog(wichObject, level, message, source, avoidDuplicates, logName)
LogManager().doLog(wichObject, level, message, source, avoidDuplicates)
def getLogs( def getLogs(
@ -166,9 +179,27 @@ class UDSLogHandler(logging.handlers.RotatingFileHandler):
Custom log handler that will log to database before calling to RotatingFileHandler Custom log handler that will log to database before calling to RotatingFileHandler
""" """
def emit(self, record: logging.LogRecord) -> None: # Protects from recursive calls
# Currently, simply call to parent emiting: typing.ClassVar[bool] = False
msg = self.format(record) # pylint: disable=unused-variable
def emit(self, record: logging.LogRecord) -> None:
if apps.ready and record.levelno > logging.INFO and not UDSLogHandler.emiting:
try:
UDSLogHandler.emiting = True
msg = self.format(record)
# Remove date and time from message, as it will be stored on database
msg = DATETIME_PATTERN.sub('', msg)
doLog(
None,
LogLevel.fromInt(record.levelno * 1000),
msg,
LogSource.LOGS,
False,
os.path.basename(self.baseFilename)
)
except Exception: # nosec: If cannot log, just ignore it
pass
finally:
UDSLogHandler.emiting = False
# TODO: Log message on database and continue as a RotatingFileHandler
return super().emit(record) return super().emit(record)

34
server/src/uds/transports/HTML5SSH/html5ssh.py
View File

@ -79,20 +79,20 @@ class HTML5SSHTransport(transports.Transport):
defvalue='https://', defvalue='https://',
length=64, length=64,
required=True, required=True,
tab=gui.TUNNEL_TAB, tab=gui.Tab.TUNNEL,
) )
username = gui.TextField( username = gui.TextField(
label=_('Username'), label=_('Username'),
order=20, order=20,
tooltip=_('Username for SSH connection authentication.'), tooltip=_('Username for SSH connection authentication.'),
tab=gui.CREDENTIALS_TAB, tab=gui.Tab.CREDENTIALS,
) )
password = gui.PasswordField( password = gui.PasswordField(
label=_('Password'), label=_('Password'),
order=21, order=21,
tooltip=_('Password for SSH connection authentication'), tooltip=_('Password for SSH connection authentication'),
tab=gui.CREDENTIALS_TAB, tab=gui.Tab.CREDENTIALS,
) )
sshPrivateKey = gui.TextField( sshPrivateKey = gui.TextField(
label=_('SSH Private Key'), label=_('SSH Private Key'),
@ -109,7 +109,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_( tooltip=_(
'Passphrase for SSH private key if it is required. If not provided, but it is needed, user will be prompted for it.' 'Passphrase for SSH private key if it is required. If not provided, but it is needed, user will be prompted for it.'
), ),
tab=gui.CREDENTIALS_TAB, tab=gui.Tab.CREDENTIALS,
) )
sshCommand = gui.TextField( sshCommand = gui.TextField(
@ -118,7 +118,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_( tooltip=_(
'Command to execute on the remote server. If not provided, an interactive shell will be executed.' 'Command to execute on the remote server. If not provided, an interactive shell will be executed.'
), ),
tab=gui.PARAMETERS_TAB, tab=gui.Tab.PARAMETERS,
) )
enableFileSharing = gui.ChoiceField( enableFileSharing = gui.ChoiceField(
label=_('File Sharing'), label=_('File Sharing'),
@ -131,7 +131,7 @@ class HTML5SSHTransport(transports.Transport):
{'id': 'up', 'text': _('Allow upload only')}, {'id': 'up', 'text': _('Allow upload only')},
{'id': 'true', 'text': _('Enable file sharing')}, {'id': 'true', 'text': _('Enable file sharing')},
], ],
tab=gui.PARAMETERS_TAB, tab=gui.Tab.PARAMETERS,
) )
fileSharingRoot = gui.TextField( fileSharingRoot = gui.TextField(
label=_('File Sharing Root'), label=_('File Sharing Root'),
@ -139,7 +139,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_( tooltip=_(
'Root path for file sharing. If not provided, root directory will be used.' 'Root path for file sharing. If not provided, root directory will be used.'
), ),
tab=gui.PARAMETERS_TAB, tab=gui.Tab.PARAMETERS,
) )
sshPort = gui.NumericField( sshPort = gui.NumericField(
length=40, length=40,
@ -148,7 +148,7 @@ class HTML5SSHTransport(transports.Transport):
order=33, order=33,
tooltip=_('Port of the SSH server.'), tooltip=_('Port of the SSH server.'),
required=True, required=True,
tab=gui.PARAMETERS_TAB, tab=gui.Tab.PARAMETERS,
) )
sshHostKey = gui.TextField( sshHostKey = gui.TextField(
label=_('SSH Host Key'), label=_('SSH Host Key'),
@ -156,7 +156,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_( tooltip=_(
'Host key of the SSH server. If not provided, no verification of host identity is done.' 'Host key of the SSH server. If not provided, no verification of host identity is done.'
), ),
tab=gui.PARAMETERS_TAB, tab=gui.Tab.PARAMETERS,
) )
serverKeepAlive = gui.NumericField( serverKeepAlive = gui.NumericField(
length=3, length=3,
@ -168,7 +168,7 @@ class HTML5SSHTransport(transports.Transport):
), ),
required=True, required=True,
minValue=0, minValue=0,
tab=gui.PARAMETERS_TAB, tab=gui.Tab.PARAMETERS,
) )
ticketValidity = gui.NumericField( ticketValidity = gui.NumericField(
@ -181,7 +181,7 @@ class HTML5SSHTransport(transports.Transport):
), ),
required=True, required=True,
minValue=60, minValue=60,
tab=gui.ADVANCED_TAB, tab=gui.Tab.ADVANCED,
) )
forceNewWindow = gui.ChoiceField( forceNewWindow = gui.ChoiceField(
order=91, order=91,
@ -202,7 +202,7 @@ class HTML5SSHTransport(transports.Transport):
), ),
], ],
defvalue=gui.FALSE, defvalue=gui.FALSE,
tab=gui.ADVANCED_TAB, tab=gui.Tab.ADVANCED
) )
def initialize(self, values: 'Module.ValuesType'): def initialize(self, values: 'Module.ValuesType'):
@ -231,13 +231,13 @@ class HTML5SSHTransport(transports.Transport):
def getLink( def getLink(
self, self,
userService: 'models.UserService', userService: 'models.UserService', # pylint: disable=unused-argument
transport: 'models.Transport', transport: 'models.Transport',
ip: str, ip: str,
os: 'DetectedOsInfo', os: 'DetectedOsInfo', # pylint: disable=unused-argument
user: 'models.User', user: 'models.User', # pylint: disable=unused-argument
password: str, password: str, # pylint: disable=unused-argument
request: 'ExtendedHttpRequestWithUser', request: 'ExtendedHttpRequestWithUser', # pylint: disable=unused-argument
) -> str: ) -> str:
# Build params dict # Build params dict
params = { params = {