shaba/openuds
1
0
Fork 1
mirror of https://github.com/dkmstr/openuds.git synced 2025-01-08 21:18:00 +03:00
Code Issues Releases Wiki Activity

Adding Database logs storage (appart from local files)

Browse Source
This commit is contained in:
Adolfo Gómez García 2023-04-20 04:19:57 +02:00
parent 9030ff3ab3
commit 06a598d577
No known key found for this signature in database
GPG Key ID: DD1ABF20724CDA23
7 changed files with 101 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/uds/REST/log.py
View File

@ -48,6 +48,7 @@ if typing.TYPE_CHECKING:
UUID_REPLACER = (
('providers', models.Provider),
('services', models.Service),
('servicespools', models.ServicePool),
('users', models.User),
('groups', models.Group),
)
@ -92,7 +93,7 @@ def logOperation(
doLog(
None,
level=level,
message=f'{handler.request.ip} {username}: [{handler.request.method}/{response_code}] {path}'[
message=f'{handler.request.ip}[{username}]: [{handler.request.method}/{response_code}] {path}'[
:4096
],
source=LogSource.REST,

32
server/src/uds/__init__.py
View File

@ -38,6 +38,7 @@ import logging
from django.db import connections
from django.db.backends.signals import connection_created
# from django.db.models.signals import post_migrate
from django.dispatch import receiver
@ -50,8 +51,8 @@ logger = logging.getLogger(__name__)
# Set default ssl context unverified, as MOST servers that we will connect will be with self signed certificates...
try:
_create_unverified_https_context = ssl._create_unverified_context
ssl._create_default_https_context = _create_unverified_https_context
# _create_unverified_https_context = ssl._create_unverified_context
# ssl._create_default_https_context = _create_unverified_https_context
# Capture warnnins to logg
logging.captureWarnings(True)
@ -69,17 +70,37 @@ class UDSAppConfig(AppConfig):
# with ANY command from manage.
logger.debug('Initializing app (ready) ***************')
# Now, ensures that all dynamic elements are loadad and present
# To make sure that the packages are initialized at this point
# Now, ensures that all dynamic elements are loaded and present
# To make sure that the packages are already initialized at this point
# pylint: disable=unused-import,import-outside-toplevel
from . import services
# pylint: disable=unused-import,import-outside-toplevel
from . import auths
# pylint: disable=unused-import,import-outside-toplevel
from . import mfas
# pylint: disable=unused-import,import-outside-toplevel
from . import osmanagers
# pylint: disable=unused-import,import-outside-toplevel
from . import notifiers
# pylint: disable=unused-import,import-outside-toplevel
from . import transports
# pylint: disable=unused-import,import-outside-toplevel
from . import reports
# pylint: disable=unused-import,import-outside-toplevel
from . import dispatchers
# pylint: disable=unused-import,import-outside-toplevel
from . import plugins
# pylint: disable=unused-import,import-outside-toplevel
from . import REST
# Ensure notifications table exists on local sqlite db (called "persistent" on settings.py)
@ -96,8 +117,9 @@ default_app_config = 'uds.UDSAppConfig'
# Sets up several sqlite non existing methodsm and some optimizations on sqlite
# pylint: disable=unused-argument
@receiver(connection_created)
def extend_sqlite(connection=None, **kwargs):
def extend_sqlite(connection=None, **kwargs) -> None:
if connection and connection.vendor == "sqlite":
logger.debug('Connection vendor is sqlite, extending methods')
cursor = connection.cursor()

2
server/src/uds/auths/SAML/config.py
View File

@ -4,9 +4,7 @@ from uds.core.util.config import Config
ORGANIZATION_NAME = Config.section('SAML').value('Organization Name', 'UDS', help='Organization name to display on SAML SP Metadata')
ORGANIZATION_DISPLAY = Config.section('SAML').value('Org. Display Name', 'UDS Organization', help='Organization Display name to display on SAML SP Metadata')
ORGANIZATION_URL = Config.section('SAML').value('Organization URL', 'http://www.udsenterprise.com', help='Organization url to display on SAML SP Metadata')
IDP_METADATA_CACHE = Config.section('SAML').value('IDP Metadata cache')
ORGANIZATION_NAME.get()
ORGANIZATION_DISPLAY.get()
ORGANIZATION_URL.get()
IDP_METADATA_CACHE.getInt()

5
server/src/uds/auths/SAML/saml.py
View File

@ -480,11 +480,6 @@ class SAMLAuthenticator(auths.Authenticator):
raise auths.exceptions.AuthenticatorException(
gettext('Can\'t access idp metadata')
)
self.cache.put(
'idpMetadata',
val,
config.IDP_METADATA_CACHE.getInt(True),
)
else:
val = self.idpMetadata.value

49
server/src/uds/core/managers/log/manager.py
View File

@ -29,8 +29,7 @@
"""
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import traceback
import logging
# import traceback
import typing
from uds.core.util import singleton
@ -45,9 +44,6 @@ if typing.TYPE_CHECKING:
from uds import models
logger = logging.getLogger(__name__)
class LogManager(metaclass=singleton.Singleton):
"""
Manager for logging (at database) events
@ -68,12 +64,13 @@ class LogManager(metaclass=singleton.Singleton):
message: str,
source: str,
avoidDuplicates: bool,
logName: str
):
"""
Logs a message associated to owner
"""
# Ensure message fits on space
message = str(message)[:255]
message = str(message)[:4096]
qs = Log.objects.filter(owner_id=owner_id, owner_type=owner_type.value)
# First, ensure we do not have more than requested logs, and we can put one more log item
@ -104,6 +101,7 @@ class LogManager(metaclass=singleton.Singleton):
source=source,
level=level,
data=message,
name=logName,
)
except Exception: # nosec
# Some objects will not get logged, such as System administrator objects, but this is fine
@ -134,6 +132,7 @@ class LogManager(metaclass=singleton.Singleton):
message: str,
source: str,
avoidDuplicates: bool = True,
logName: typing.Optional[str] = None,
):
"""
Do the logging for the requested object.
@ -146,25 +145,15 @@ class LogManager(metaclass=singleton.Singleton):
else LogObjectType.SYSLOG
)
objectId = getattr(wichObject, 'id', -1)
logName = logName or ''
if owner_type is not None:
try:
self._log(
owner_type, objectId, level, message, source, avoidDuplicates
owner_type, objectId, level, message, source, avoidDuplicates, logName
)
except Exception:
logger.error(
'Error logging: %s:%s %s - %s %s',
owner_type,
objectId,
level,
message,
source,
)
else:
logger.debug(
'Requested doLog for a type of object not covered: %s', wichObject
)
except Exception: # nosec
pass # Can not log,
def getLogs(
self, wichObject: typing.Optional['Model'], limit: int = -1
@ -178,7 +167,6 @@ class LogManager(metaclass=singleton.Singleton):
if wichObject
else LogObjectType.SYSLOG
)
logger.debug('Getting log: %s -> %s', wichObject, owner_type)
if owner_type: # 0 is valid owner type
return self._getLogs(
@ -187,9 +175,6 @@ class LogManager(metaclass=singleton.Singleton):
limit if limit != -1 else owner_type.get_max_elements(),
)
logger.debug(
'Requested getLogs for a type of object not covered: %s', wichObject
)
return []
def clearLogs(self, wichObject: typing.Optional['Model']):
@ -206,11 +191,11 @@ class LogManager(metaclass=singleton.Singleton):
)
if owner_type:
self._clearLogs(owner_type, getattr(wichObject, 'id', -1))
else:
logger.debug(
'Requested clearLogs for a type of object not covered: %s: %s',
type(wichObject),
wichObject,
)
for line in traceback.format_stack(limit=5):
logger.debug('>> %s', line)
#else:
# logger.debug(
# 'Requested clearLogs for a type of object not covered: %s: %s',
# type(wichObject),
# wichObject,
#)
#for line in traceback.format_stack(limit=5):
# logger.debug('>> %s', line)

45
server/src/uds/core/util/log/__init__.py
View File

@ -30,18 +30,26 @@
"""
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import os
import logging
import logging.handlers
import typing
import enum
import re
from django.apps import apps
# Not imported at runtime, just for type checking
if typing.TYPE_CHECKING:
from django.db.models import Model
logger = logging.getLogger(__name__)
useLogger = logging.getLogger('useLog')
# Patter for look for date and time in this format: 2023-04-20 04:03:08,776
# This is the format used by python logging module
DATETIME_PATTERN: typing.Final[re.Pattern] = re.compile(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})')
class LogLevel(enum.IntEnum):
OTHER = 10000
DEBUG = 20000
@ -74,6 +82,7 @@ class LogLevel(enum.IntEnum):
except ValueError:
return cls.OTHER
class LogSource(enum.StrEnum):
INTERNAL = 'internal'
ACTOR = 'actor'
@ -84,6 +93,8 @@ class LogSource(enum.StrEnum):
ADMIN = 'admin'
SERVICE = 'service'
REST = 'rest'
LOGS = 'logs'
def useLog(
type_: str,
@ -124,6 +135,8 @@ def useLog(
)
)
# Will be stored on database by UDSLogHandler
def doLog(
wichObject: typing.Optional['Model'],
@ -131,12 +144,12 @@ def doLog(
message: str,
source: LogSource = LogSource.UNKNOWN,
avoidDuplicates: bool = True,
logName: typing.Optional[str] = None,
) -> None:
# pylint: disable=import-outside-toplevel
from uds.core.managers.log import LogManager
logger.debug('%s %s %s', wichObject, level, message)
LogManager().doLog(wichObject, level, message, source, avoidDuplicates)
LogManager().doLog(wichObject, level, message, source, avoidDuplicates, logName)
def getLogs(
@ -166,9 +179,27 @@ class UDSLogHandler(logging.handlers.RotatingFileHandler):
Custom log handler that will log to database before calling to RotatingFileHandler
"""
def emit(self, record: logging.LogRecord) -> None:
# Currently, simply call to parent
msg = self.format(record) # pylint: disable=unused-variable
# Protects from recursive calls
emiting: typing.ClassVar[bool] = False
def emit(self, record: logging.LogRecord) -> None:
if apps.ready and record.levelno > logging.INFO and not UDSLogHandler.emiting:
try:
UDSLogHandler.emiting = True
msg = self.format(record)
# Remove date and time from message, as it will be stored on database
msg = DATETIME_PATTERN.sub('', msg)
doLog(
None,
LogLevel.fromInt(record.levelno * 1000),
msg,
LogSource.LOGS,
False,
os.path.basename(self.baseFilename)
)
except Exception: # nosec: If cannot log, just ignore it
pass
finally:
UDSLogHandler.emiting = False
# TODO: Log message on database and continue as a RotatingFileHandler
return super().emit(record)

34
server/src/uds/transports/HTML5SSH/html5ssh.py
View File

@ -79,20 +79,20 @@ class HTML5SSHTransport(transports.Transport):
defvalue='https://',
length=64,
required=True,
tab=gui.TUNNEL_TAB,
tab=gui.Tab.TUNNEL,
)
username = gui.TextField(
label=_('Username'),
order=20,
tooltip=_('Username for SSH connection authentication.'),
tab=gui.CREDENTIALS_TAB,
tab=gui.Tab.CREDENTIALS,
)
password = gui.PasswordField(
label=_('Password'),
order=21,
tooltip=_('Password for SSH connection authentication'),
tab=gui.CREDENTIALS_TAB,
tab=gui.Tab.CREDENTIALS,
)
sshPrivateKey = gui.TextField(
label=_('SSH Private Key'),
@ -109,7 +109,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_(
'Passphrase for SSH private key if it is required. If not provided, but it is needed, user will be prompted for it.'
),
tab=gui.CREDENTIALS_TAB,
tab=gui.Tab.CREDENTIALS,
)
sshCommand = gui.TextField(
@ -118,7 +118,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_(
'Command to execute on the remote server. If not provided, an interactive shell will be executed.'
),
tab=gui.PARAMETERS_TAB,
tab=gui.Tab.PARAMETERS,
)
enableFileSharing = gui.ChoiceField(
label=_('File Sharing'),
@ -131,7 +131,7 @@ class HTML5SSHTransport(transports.Transport):
{'id': 'up', 'text': _('Allow upload only')},
{'id': 'true', 'text': _('Enable file sharing')},
],
tab=gui.PARAMETERS_TAB,
tab=gui.Tab.PARAMETERS,
)
fileSharingRoot = gui.TextField(
label=_('File Sharing Root'),
@ -139,7 +139,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_(
'Root path for file sharing. If not provided, root directory will be used.'
),
tab=gui.PARAMETERS_TAB,
tab=gui.Tab.PARAMETERS,
)
sshPort = gui.NumericField(
length=40,
@ -148,7 +148,7 @@ class HTML5SSHTransport(transports.Transport):
order=33,
tooltip=_('Port of the SSH server.'),
required=True,
tab=gui.PARAMETERS_TAB,
tab=gui.Tab.PARAMETERS,
)
sshHostKey = gui.TextField(
label=_('SSH Host Key'),
@ -156,7 +156,7 @@ class HTML5SSHTransport(transports.Transport):
tooltip=_(
'Host key of the SSH server. If not provided, no verification of host identity is done.'
),
tab=gui.PARAMETERS_TAB,
tab=gui.Tab.PARAMETERS,
)
serverKeepAlive = gui.NumericField(
length=3,
@ -168,7 +168,7 @@ class HTML5SSHTransport(transports.Transport):
),
required=True,
minValue=0,
tab=gui.PARAMETERS_TAB,
tab=gui.Tab.PARAMETERS,
)
ticketValidity = gui.NumericField(
@ -181,7 +181,7 @@ class HTML5SSHTransport(transports.Transport):
),
required=True,
minValue=60,
tab=gui.ADVANCED_TAB,
tab=gui.Tab.ADVANCED,
)
forceNewWindow = gui.ChoiceField(
order=91,
@ -202,7 +202,7 @@ class HTML5SSHTransport(transports.Transport):
),
],
defvalue=gui.FALSE,
tab=gui.ADVANCED_TAB,
tab=gui.Tab.ADVANCED
)
def initialize(self, values: 'Module.ValuesType'):
@ -231,13 +231,13 @@ class HTML5SSHTransport(transports.Transport):
def getLink(
self,
userService: 'models.UserService',
userService: 'models.UserService', # pylint: disable=unused-argument
transport: 'models.Transport',
ip: str,
os: 'DetectedOsInfo',
user: 'models.User',
password: str,
request: 'ExtendedHttpRequestWithUser',
os: 'DetectedOsInfo', # pylint: disable=unused-argument
user: 'models.User', # pylint: disable=unused-argument
password: str, # pylint: disable=unused-argument
request: 'ExtendedHttpRequestWithUser', # pylint: disable=unused-argument
) -> str:
# Build params dict
params = {