shaba/openuds
1
0
Fork 1
mirror of https://github.com/dkmstr/openuds.git synced 2025-10-14 15:33:43 +03:00
Code Issues Releases Wiki Activity

Adding more server REST api tests and fixes

Browse Source
This commit is contained in:
Adolfo Gómez García
2023-09-06 18:33:00 +02:00
parent dda77ab2a3
commit dc901b4473
22 changed files with 509 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
server/src/tests/REST/servers/test_events.py Normal file
View File

@@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2022 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import typing
import logging
from unittest import mock
from uds.core.util import log
from ...utils import rest, random_ip_v4, random_ip_v6, random_mac
from ...fixtures import servers as servers_fixtures
if typing.TYPE_CHECKING:
from ...utils.test import UDSHttpResponse
logger = logging.getLogger(__name__)
class ServerEventsTest(rest.test.RESTTestCase):
"""
Test server functionality
"""
def test_event(self) -> None:
pass

131
server/src/tests/REST/servers/test_events_log.py Normal file
View File

@@ -0,0 +1,131 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2022 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import typing
import logging
from unittest import mock
from uds.core.util import log
from ...utils import rest, random_ip_v4, random_ip_v6, random_mac
from ...fixtures import servers as servers_fixtures
if typing.TYPE_CHECKING:
from ...utils.test import UDSHttpResponse
logger = logging.getLogger(__name__)
class ServerEventsLogTest(rest.test.RESTTestCase):
"""
Test server functionality
"""
def test_event_log(self) -> None:
# REST path: servers/notify (/uds/rest/...)
# Log data:
# logData = {
# 'token': 'server token', # Must be present on all events
# 'type': 'log',
# 'user_service': 'optional userService uuid', if not present, is a log for the server of the token
# 'level': 'debug|info'|'warning'|'error',
# 'message': 'message',
# }
server = servers_fixtures.createServer()
userService = self.user_service_managed
self.login() # Login as staff
# Mock the "log.doLog" method (uds.core.util.log.doLog)
with mock.patch('uds.core.managers.log.manager.LogManager.doLog') as doLog:
# Now notify to server
response = self.client.rest_post(
'/servers/event',
data={
'token': server.token,
'type': 'log',
'level': 'info',
'message': 'test message',
},
)
self.assertEqual(response.status_code, 200)
# First call shout have
doLog.assert_any_call(server, log.LogLevel.INFO, 'test message', log.LogSource.SERVER, None)
# Now notify to an userService
response = self.client.rest_post(
'servers/event',
data={
'token': server.token,
'user_service': userService.uuid,
'type': 'log',
'level': 'info',
'message': 'test message userservice',
},
)
self.assertEqual(response.status_code, 200)
doLog.assert_any_call(
userService, log.LogLevel.INFO, 'test message userservice', log.LogSource.SERVER, None
)
def test_event_log_fail(self) -> None:
server = servers_fixtures.createServer()
self.login()
data = {
'token': server.token,
'type': 'log',
'level': 'info',
'message': 'test',
}
for field, value in (
('token', None),
('type', 'invalid'),
# Invalid level should log as "other"
('message', None),
):
fail_data = data.copy()
if value is None:
del fail_data[field]
else:
fail_data[field] = value
response = self.client.rest_post(
'/servers/event',
data=fail_data,
)
if field == 'token':
self.assertEqual(response.status_code, 400, f'Error on field {field}')
else:
self.assertEqual(response.status_code, 200, f'Error on field {field}')
self.assertIsNotNone(response.content, f'Error not found for field {field}')
self.assertIsNotNone(response.json(), f'Error not found for field {field}')
self.assertIn('error', response.json(), f'Error not found for field {field}')

53
server/src/tests/REST/servers/test_events_login_logout.py Normal file
View File

@@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2022 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import typing
import logging
from unittest import mock
from uds.core.util import log
from ...utils import rest, random_ip_v4, random_ip_v6, random_mac
from ...fixtures import servers as servers_fixtures
if typing.TYPE_CHECKING:
from ...utils.test import UDSHttpResponse
logger = logging.getLogger(__name__)
class ServerEventsLoginLogoutTest(rest.test.RESTTestCase):
"""
Test server functionality
"""
def test_login(self) -> None:
pass

12
server/src/tests/REST/servers/test_register.py
View File

@@ -172,4 +172,16 @@ class ServerRegisterTest(rest.test.RESTTestCase):
self._data['mac'] = random_mac()
self._data['data'] = 'invalid json'
_do_test('invalid json')
def test_invalid_user_not_staff_or_admin(self) -> None:
self.login(self.plain_users[0])
# Login successfull, but not admin or staff
# Data is invalid, but we will get a 403 because we are not admin or staff
response = self.client.rest_post(
'servers/register',
data=self._data,
content_type='application/json',
)
self.assertEqual(response.status_code, 403)
self.assertIn('denied', response.content.decode().lower())

65
server/src/tests/REST/servers/test_test.py Normal file
View File

@@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2022 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import typing
import logging
from uds import models
from uds.core import types
from uds.core.managers import crypto
from uds.core.util import log
from ...utils import rest, random_ip_v4, random_ip_v6, random_mac
from ...fixtures import servers as servers_fixtures
if typing.TYPE_CHECKING:
from ...utils.test import UDSHttpResponse
logger = logging.getLogger(__name__)
class ServerTestTest(rest.test.RESTTestCase):
"""
Test server functionality
"""
def test_server_test(self) -> None:
"""
Test server rest api registration
"""
server = servers_fixtures.createServer()
response = self.client.rest_post(
'servers/test',
data={
'token': server.token,
},
)
self.assertEqual(response.status_code, 200)

2
server/src/tests/core/managers/test_servers_managed.py
View File

@@ -107,7 +107,7 @@ class ServerManagerManagedServersTest(UDSTestCase):
typing.Callable[['models.Server'], typing.Optional['types.servers.ServerStatsType']]
] = None,
) -> typing.Iterator[mock.Mock]:
with mock.patch('uds.core.managers.servers_api.request.ServerApiRequester') as mockServerApiRequester:
with mock.patch('uds.core.managers.servers_api.requester.ServerApiRequester') as mockServerApiRequester:
def _getStats() -> typing.Optional[types.servers.ServerStatsType]:
# Get first argument from call to init on serverApiRequester

2
server/src/tests/core/managers/test_servers_unmanaged.py
View File

@@ -86,7 +86,7 @@ class ServerManagerUnmanagedServersTest(UDSTestCase):
@contextmanager
def createMockApiRequester(self) -> typing.Iterator[mock.Mock]:
with mock.patch('uds.core.managers.servers_api.request.ServerApiRequester') as mockServerApiRequester:
with mock.patch('uds.core.managers.servers_api.requester.ServerApiRequester') as mockServerApiRequester:
mockServerApiRequester.return_value.getStats.return_value = None
yield mockServerApiRequester

5
server/src/tests/fixtures/servers.py vendored
View File

@@ -39,13 +39,14 @@ from ..utils import generators
def createServer(
type: 'types.servers.ServerType',
type: 'types.servers.ServerType' = types.servers.ServerType.SERVER,
subtype: typing.Optional[str] = None,
version: typing.Optional[str] = None,
ip: typing.Optional[str] = None,
listen_port: int = 0,
data: typing.Any = None,
) -> 'models.Server':
# Token is created by default on record creation
return models.Server.objects.create(
username=generators.random_string(),
ip_from=ip or '127.0.0.1',
@@ -62,7 +63,7 @@ def createServer(
def createServerGroup(
type: 'types.servers.ServerType',
type: 'types.servers.ServerType' = types.servers.ServerType.SERVER,
subtype: typing.Optional[str] = None,
version: typing.Optional[str] = None,
ip: typing.Optional[str] = None,

46
server/src/tests/utils/rest/test.py
View File

@@ -50,9 +50,9 @@ class RESTTestCase(test.UDSTransactionTestCase):
auth: models.Authenticator
simple_groups: typing.List[models.Group]
meta_groups: typing.List[models.Group]
admins: typing.List[models.User]
staffs: typing.List[models.User]
plain_users: typing.List[models.User]
admins: typing.List[models.User] # Users that are admin
staffs: typing.List[models.User] # Users that are not admin but are staff
plain_users: typing.List[models.User] # Users that are not admin or staff
users = property(lambda self: self.admins + self.staffs + self.plain_users)
groups = property(lambda self: self.simple_groups + self.meta_groups)
@@ -62,19 +62,15 @@ class RESTTestCase(test.UDSTransactionTestCase):
user_service_unmanaged: models.UserService
user_services: typing.List[models.UserService]
auth_token: str = ''
def setUp(self) -> None:
# Set up data for REST Test cases
# First, the authenticator related
self.auth = authenticators_fixtures.createAuthenticator()
self.simple_groups = authenticators_fixtures.createGroups(
self.auth, NUMBER_OF_ITEMS_TO_CREATE
)
self.meta_groups = authenticators_fixtures.createMetaGroups(
self.auth, NUMBER_OF_ITEMS_TO_CREATE
)
self.simple_groups = authenticators_fixtures.createGroups(self.auth, NUMBER_OF_ITEMS_TO_CREATE)
self.meta_groups = authenticators_fixtures.createMetaGroups(self.auth, NUMBER_OF_ITEMS_TO_CREATE)
# Create some users, one admin, one staff and one user
self.admins = authenticators_fixtures.createUsers(
self.auth,
@@ -106,21 +102,17 @@ class RESTTestCase(test.UDSTransactionTestCase):
self.groups,
'managed',
)
self.user_service_unmanaged = (
services_fixtures.createOneCacheTestingUserService(
self.provider,
self.admins[0],
self.groups,
'unmanaged',
)
self.user_service_unmanaged = services_fixtures.createOneCacheTestingUserService(
self.provider,
self.admins[0],
self.groups,
'unmanaged',
)
self.user_services = []
for user in self.users:
self.user_services.append(
services_fixtures.createOneCacheTestingUserService(
self.provider, user, self.groups, 'managed'
)
services_fixtures.createOneCacheTestingUserService(self.provider, user, self.groups, 'managed')
)
self.user_services.append(
services_fixtures.createOneCacheTestingUserService(
@@ -128,9 +120,7 @@ class RESTTestCase(test.UDSTransactionTestCase):
)
)
def login(
self, user: typing.Optional[models.User] = None, as_admin: bool = True
) -> None:
def login(self, user: typing.Optional[models.User] = None, as_admin: bool = True) -> None:
'''
Login as specified and returns the auth token
The token is inserted on the header of the client, so it can be used in the rest of the tests
@@ -148,19 +138,15 @@ class RESTTestCase(test.UDSTransactionTestCase):
# Insert token into headers
self.client.add_header(AUTH_TOKEN_HEADER, response['token'])
self.auth_token = response['token']
class RESTActorTestCase(RESTTestCase):
# Login as admin or staff and register an actor
# Returns as a tuple the auth token and the actor registration result token:
# - The login auth token
# - The actor token
def login_and_register(self, as_admin: bool = True) -> str:
self.login(
as_admin=as_admin
) # Token not used, alreade inserted on login
self.login(as_admin=as_admin) # Token not used, alreade inserted on login
response = self.client.post(
'/uds/rest/actor/v3/register',
data=self.register_data(constants.STRING_CHARS),
@@ -169,9 +155,7 @@ class RESTActorTestCase(RESTTestCase):
self.assertEqual(response.status_code, 200, 'Actor registration failed')
return response.json()['result']
def register_data(
self, chars: typing.Optional[str] = None
) -> typing.Dict[str, str]:
def register_data(self, chars: typing.Optional[str] = None) -> typing.Dict[str, str]:
# Data for registration
return {
'username': generators.random_string(size=12, chars=chars)

5
server/src/tests/utils/test.py
View File

@@ -136,6 +136,7 @@ class UDSClient(UDSClientMixin, Client):
def rest_post(self, method: str, *args, **kwargs) -> 'UDSHttpResponse':
# compose url
kwargs['content_type'] = kwargs.get('content_type', 'application/json')
return self.post(self.compose_rest_url(method), *args, **kwargs)
def put(self, *args, **kwargs) -> 'UDSHttpResponse':
@@ -250,8 +251,6 @@ class UDSTransactionTestCase(UDSTestCaseMixin, TransactionTestCase):
# pylint: disable=unused-argument
def setupClass(
cls: typing.Union[typing.Type[UDSTestCase], typing.Type[UDSTransactionTestCase]]
) -> None:
def setupClass(cls: typing.Union[typing.Type[UDSTestCase], typing.Type[UDSTransactionTestCase]]) -> None:
# Nothing right now
pass

4
server/src/uds/REST/dispatcher.py
View File

@@ -149,8 +149,8 @@ class Dispatcher(View):
logger.debug('Path: %s', full_path)
logger.debug('Error: %s', e)
log.logOperation(handler, 500, log.LogLevel.ERROR)
return http.HttpResponseServerError(
log.logOperation(handler, 400, log.LogLevel.ERROR)
return http.HttpResponseBadRequest(
f'Invalid parameters invoking {full_path}: {e}',
content_type="text/plain",
)

1
server/src/uds/REST/log.py
View File

@@ -97,5 +97,4 @@ def logOperation(
:4096
],
source=LogSource.REST,
avoidDuplicates=False,
)

64
server/src/uds/REST/methods/servers.py
View File

@@ -45,6 +45,7 @@ logger = logging.getLogger(__name__)
# REST API for Server Token Clients interaction
# Register is split in two because tunnel registration also uses this
class ServerRegisterBase(Handler):
def post(self) -> typing.MutableMapping[str, typing.Any]:
serverToken: models.Server
@@ -58,7 +59,7 @@ class ServerRegisterBase(Handler):
data = self._params.get('data', None)
subtype = self._params.get('subtype', '')
os = self._params.get('os', KnownOS.UNKNOWN.os_name()).lower()
type = self._params['type'] # MUST be present
hostname = self._params['hostname'] # MUST be present
# Validate parameters
@@ -139,25 +140,7 @@ class ServerTest(Handler):
return rest_result('error', error=str(e))
# Server related classes/actions
class ServerAction(Handler):
authenticated = False # Actor requests are not authenticated normally
path = 'servers/action'
def action(self, server: models.Server) -> typing.MutableMapping[str, typing.Any]:
return rest_result('error', error='Base action invoked')
@decorators.blocker()
def post(self) -> typing.MutableMapping[str, typing.Any]:
try:
server = models.Server.objects.get(token=self._params['token'])
except models.Server.DoesNotExist:
raise exceptions.BlockAccess() from None # Block access if token is not valid
return self.action(server)
class ServerEvent(ServerAction):
class ServerEvent(Handler):
"""
Manages a event notification from a server to UDS Broker
@@ -169,7 +152,9 @@ class ServerEvent(ServerAction):
* log
"""
name = 'notify'
authenticated = False # Actor requests are not authenticated normally
path = 'servers'
name = 'event'
def getUserService(self) -> models.UserService:
'''
@@ -181,7 +166,17 @@ class ServerEvent(ServerAction):
logger.error('User service not found (params: %s)', self._params)
raise
def action(self, server: models.Server) -> typing.MutableMapping[str, typing.Any]:
@decorators.blocker()
def post(self) -> typing.MutableMapping[str, typing.Any]:
# Avoid circular import
from uds.core.managers.servers import ServerManager
try:
server = models.Server.objects.get(token=self._params['token'])
except models.Server.DoesNotExist:
raise exceptions.BlockAccess() from None # Block access if token is not valid
except KeyError:
raise rest_exceptions.RequestError('Token not present') from None # Invalid request if token is not present
# Notify a server that a new service has been assigned to it
# Get action from parameters
# Parameters:
@@ -192,24 +187,7 @@ class ServerEvent(ServerAction):
# * Logout: { 'username': 'username'}
# * Log: { 'level': 'level', 'message': 'message'}
try:
event = types.events.NotifiableEvents(self._params.get('event', None) or '')
except ValueError:
return rest_result('error', error='No valid event specified')
# Extract user service
try:
userService = self.getUserService()
except Exception:
return rest_result('error', error='User service not found')
if event == types.events.NotifiableEvents.LOGIN:
# TODO: notify
pass
elif event == types.events.NotifiableEvents.LOGOUT:
# TODO: notify
pass
elif event == types.events.NotifiableEvents.LOG:
# TODO: log
pass
return rest_result(True)
return ServerManager.manager().processEvent(server, self._params)
except Exception as e:
logger.error('Error processing event %s: %s', self._params, e)
return rest_result('error', error='Error processing event')

13
server/src/uds/core/managers/log/manager.py
View File

@@ -64,7 +64,6 @@ class LogManager(metaclass=singleton.Singleton):
level: int,
message: str,
source: str,
avoidDuplicates: bool,
logName: str
):
"""
@@ -75,14 +74,6 @@ class LogManager(metaclass=singleton.Singleton):
qs = Log.objects.filter(owner_id=owner_id, owner_type=owner_type.value)
if avoidDuplicates:
lg: typing.Optional['Log'] = Log.objects.filter(
owner_id=owner_id, owner_type=owner_type.value
).last()
if lg and lg.data == message:
# Do not log again, already logged
return
# now, we add new log
try:
Log.objects.create(
@@ -122,9 +113,7 @@ class LogManager(metaclass=singleton.Singleton):
level: int,
message: str,
source: str,
avoidDuplicates: bool = True,
logName: typing.Optional[str] = None,
delayInsert: bool = False,
):
"""
Do the logging for the requested object.
@@ -142,7 +131,7 @@ class LogManager(metaclass=singleton.Singleton):
if owner_type is not None:
try:
self._log(
owner_type, objectId, level, message, source, avoidDuplicates, logName
owner_type, objectId, level, message, source, logName
)
except Exception: # nosec
pass # Can not log,

8
server/src/uds/core/managers/log/objects.py
View File

@@ -1,4 +1,5 @@
import typing
import functools
import enum
from uds import models
@@ -7,7 +8,7 @@ from uds import models
if typing.TYPE_CHECKING:
from django.db.models import Model
# Note: Once assigned a value, do not change it, as it will break the log
class LogObjectType(enum.IntEnum):
USERSERVICE = 0
PUBLICATION = 1
@@ -19,10 +20,12 @@ class LogObjectType(enum.IntEnum):
AUTHENTICATOR = 7
METAPOOL = 8
SYSLOG = 9
SERVER = 10
@functools.lru_cache(maxsize=16)
def get_max_elements(self) -> int:
"""
if True, this type of log will be limited by number of log entries
Returns the max number of elements to be stored for this type of log
"""
from uds.core.util.config import GlobalConfig # pylint: disable=import-outside-toplevel
@@ -36,6 +39,7 @@ MODEL_TO_TYPE: typing.Mapping[typing.Type['Model'], LogObjectType] = {
models.ServicePoolPublication: LogObjectType.PUBLICATION,
models.ServicePool: LogObjectType.SERVICEPOOL,
models.Service: LogObjectType.SERVICE,
models.Server: LogObjectType.SERVER,
models.Provider: LogObjectType.PROVIDER,
models.User: LogObjectType.USER,
models.Group: LogObjectType.GROUP,

9
server/src/uds/core/managers/notifications.py
View File

@@ -67,6 +67,9 @@ class NotificationsManager(metaclass=singleton.Singleton):
message = message[:4096] # Max length of message
# Store the notification on local persistent storage
# Will be processed by UDS backend
with Notification.atomicPersistent():
notify = Notification(group=group, identificator=identificator, level=level, message=message)
Notification.savePersistent(notify)
try:
with Notification.atomicPersistent():
notify = Notification(group=group, identificator=identificator, level=level, message=message)
notify.savePersistent()
except Exception:
logger.info('Error saving notification %s, %s, %s, %s', group, identificator, level, message)

52
server/src/uds/core/managers/servers.py
View File

@@ -44,7 +44,7 @@ from uds.core.util import model as model_utils
from uds.core.util import singleton
from uds.core.util.storage import StorageAccess, Storage
from .servers_api import request
from .servers_api import events, requester
logger = logging.getLogger(__name__)
traceLogger = logging.getLogger('traceLog')
@@ -116,19 +116,24 @@ class ServerManager(metaclass=singleton.Singleton):
fltrs = fltrs.filter(Q(locked_until=None) | Q(locked_until__lte=now)) # Only unlocked servers
if excludeServersUUids:
fltrs = fltrs.exclude(uuid__in=excludeServersUUids)
# Paralelize stats retrieval
cachedStats: typing.List[typing.Tuple[typing.Optional['types.servers.ServerStatsType'], 'models.Server']] = []
cachedStats: typing.List[
typing.Tuple[typing.Optional['types.servers.ServerStatsType'], 'models.Server']
] = []
def _retrieveStats(server: 'models.Server') -> None:
try:
cachedStats.append((request.ServerApiRequester(server).getStats(), server)) # Store stats for later use
cachedStats.append(
(requester.ServerApiRequester(server).getStats(), server)
) # Store stats for later use
except Exception:
cachedStats.append((None, server))
with ThreadPoolExecutor(max_workers=10) as executor:
for server in fltrs.select_for_update():
executor.submit(_retrieveStats, server)
# Now, cachedStats has a list of tuples (stats, server), use it to find the best server
for stats, server in cachedStats:
if stats is None:
@@ -162,7 +167,7 @@ class ServerManager(metaclass=singleton.Singleton):
# If best was locked, notify it (will be notified again on assign)
if best[0].locked_until is not None:
request.ServerApiRequester(best[0]).notifyRelease(userService)
requester.ServerApiRequester(best[0]).notifyRelease(userService)
return best
@@ -198,7 +203,7 @@ class ServerManager(metaclass=singleton.Singleton):
# Look for existint user asignation through properties
prop_name = self.propertyName(userService.user)
now = model_utils.getSqlDatetime()
excludeServersUUids = excludeServersUUids or set()
with serverGroup.properties as props:
@@ -207,7 +212,10 @@ class ServerManager(metaclass=singleton.Singleton):
] = types.servers.ServerCounterType.fromIterable(props.get(prop_name))
# If server is forced, and server is part of the group, use it
if server:
if server.groups.filter(uuid=serverGroup.uuid).exclude(uuid__in=excludeServersUUids).count() == 0:
if (
server.groups.filter(uuid=serverGroup.uuid).exclude(uuid__in=excludeServersUUids).count()
== 0
):
raise exceptions.UDSException(_('Server is not part of the group'))
elif server.maintenance_mode:
raise exceptions.UDSException(_('Server is in maintenance mode'))
@@ -259,7 +267,7 @@ class ServerManager(metaclass=singleton.Singleton):
# Notify assgination in every case, even if reassignation to same server is made
# This lets the server to keep track, if needed, of multi-assignations
request.ServerApiRequester(bestServer).notifyAssign(userService, serviceType, info.counter)
requester.ServerApiRequester(bestServer).notifyAssign(userService, serviceType, info.counter)
return info
def release(
@@ -288,13 +296,17 @@ class ServerManager(metaclass=singleton.Singleton):
with transaction.atomic():
resetCounter = False
# ServerCounterType
serverCounter: typing.Optional[types.servers.ServerCounterType] = types.servers.ServerCounterType.fromIterable(props.get(prop_name))
serverCounter: typing.Optional[
types.servers.ServerCounterType
] = types.servers.ServerCounterType.fromIterable(props.get(prop_name))
# If no cached value, get server assignation
if serverCounter is None:
return types.servers.ServerCounterType.empty()
# Ensure counter is at least 1
serverCounter = types.servers.ServerCounterType(serverCounter.server_uuid, max(1, serverCounter.counter))
serverCounter = types.servers.ServerCounterType(
serverCounter.server_uuid, max(1, serverCounter.counter)
)
if serverCounter.counter == 1 or unlock:
# Last one, remove it
del props[prop_name]
@@ -314,20 +326,20 @@ class ServerManager(metaclass=singleton.Singleton):
if server.type == types.servers.ServerType.UNMANAGED:
self.decreaseUnmanagedUsage(server.uuid, forceReset=resetCounter)
request.ServerApiRequester(server).notifyRelease(userService)
requester.ServerApiRequester(server).notifyRelease(userService)
return types.servers.ServerCounterType(serverCounter.server_uuid, serverCounter.counter - 1)
def getAssignInformation(self, serverGroup: 'models.ServerGroup') -> typing.Dict[str, int]:
"""
Get usage information for a server group
Args:
serverGroup: Server group to get current usage from
Returns:
Dict of current usage (user uuid, counter for assignations to that user)
"""
res: typing.Dict[str, int] = {}
for k, v in serverGroup.properties.items():
@@ -363,10 +375,12 @@ class ServerManager(metaclass=singleton.Singleton):
"""
Notifies preconnect to server
"""
request.ServerApiRequester(server).notifyPreconnect(userService, info)
requester.ServerApiRequester(server).notifyPreconnect(userService, info)
def processNotification(self, server: 'models.Server', data: str) -> None:
def processEvent(self, server: 'models.Server', data: typing.Dict[str, typing.Any]) -> typing.Any:
"""
Processes a notification FROM server
That is, this is not invoked directly unless a REST request is received from
a server.
"""
pass
return events.process(data)

93
server/src/uds/core/managers/servers_api/events.py Normal file
View File

@@ -0,0 +1,93 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2023 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
Author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import logging
import typing
from uds.core import types, consts
from uds.REST.utils import rest_result
from uds import models
from uds.core.util import log
logger = logging.getLogger(__name__)
def process_log(data: typing.Dict[str, typing.Any]) -> typing.Any:
if 'user_service' in data: # Log for an user service
userService = models.UserService.objects.get(uuid=data['user_service'])
log.doLog(
userService, log.LogLevel.fromStr(data['level']), data['message'], source=log.LogSource.SERVER
)
else:
server = models.Server.objects.get(token=data['token'])
log.doLog(server, log.LogLevel.fromStr(data['level']), data['message'], source=log.LogSource.SERVER)
return rest_result(consts.OK)
def process_login(data: typing.Dict[str, typing.Any]) -> typing.Any:
return rest_result(consts.OK)
def process_logout(data: typing.Dict[str, typing.Any]) -> typing.Any:
return rest_result(consts.OK)
def process_ping(data: typing.Dict[str, typing.Any]) -> typing.Any:
return rest_result(consts.OK)
PROCESSORS: typing.Final[typing.Mapping[str, typing.Callable[[typing.Dict[str, typing.Any]], typing.Any]]] = {
'log': process_log,
'login': process_login,
'logout': process_logout,
'ping': process_ping,
}
def process(data: typing.Dict[str, typing.Any]) -> typing.Any:
"""Processes the event data
Valid events are (in key 'type'):
* log: A log message (to server or userService)
* login: A login has been made (to an userService)
* logout: A logout has been made (to an userService)
* ping: A ping request (can include stats, etc...)
"""
try:
fnc = PROCESSORS[data['type']]
except KeyError:
logger.error('Invalid event type: %s', data.get('type', 'not_found'))
return rest_result('error', error=f'Invalid event type {data.get("type", "not_found")}')
try:
fnc(data)
except Exception as e:
logger.error('Exception processing event %s: %s', data, e)
return rest_result('error', error=str(e))

2
server/src/uds/core/managers/servers_api/request.py → server/src/uds/core/managers/servers_api/requester.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2019-2021 Virtual Cable S.L.U.
# Copyright (c) 2023 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,

15
server/src/uds/core/util/log.py
View File

@@ -126,6 +126,7 @@ class LogSource(enum.StrEnum):
WEB = 'web'
ADMIN = 'admin'
SERVICE = 'service'
SERVER = 'server'
REST = 'rest'
LOGS = 'logs'
@@ -177,14 +178,12 @@ def doLog(
level: LogLevel,
message: str,
source: LogSource = LogSource.UNKNOWN,
avoidDuplicates: bool = True,
logName: typing.Optional[str] = None,
delayInsert: bool = False,
) -> None:
# pylint: disable=import-outside-toplevel
from uds.core.managers.log import LogManager
LogManager.manager().doLog(wichObject, level, message, source, avoidDuplicates, logName, delayInsert=delayInsert)
LogManager.manager().doLog(wichObject, level, message, source, logName)
def getLogs(wichObject: typing.Optional['Model'], limit: int = -1) -> typing.List[typing.Dict]:
@@ -220,11 +219,11 @@ class UDSLogHandler(logging.handlers.RotatingFileHandler):
# pylint: disable=import-outside-toplevel
from uds.core.managers.notifications import NotificationsManager
def getMsg(*, removeLevel: bool) -> str:
def formatMessage(*, clearLevel: bool) -> str:
msg = self.format(record)
# Remove date and time from message, as it will be stored on database
msg = DATETIME_PATTERN.sub('', msg)
if removeLevel:
if clearLevel:
# Remove log level from message, as it will be stored on database
msg = LOGLEVEL_PATTERN.sub('', msg)
return msg
@@ -238,11 +237,11 @@ class UDSLogHandler(logging.handlers.RotatingFileHandler):
logLevel = LogLevel.fromLoggingLevel(record.levelno)
UDSLogHandler.emiting = True
identificator = os.path.basename(self.baseFilename)
msg = getMsg(removeLevel=True)
msg = formatMessage(clearLevel=True)
if record.levelno >= logging.WARNING:
# Remove traceback from message, as it will be stored on database
notify(msg.splitlines()[0], identificator, logLevel)
doLog(None, logLevel, msg, LogSource.LOGS, False, identificator, delayInsert=True)
doLog(None, logLevel, msg, LogSource.LOGS, identificator)
except Exception: # nosec: If cannot log, just ignore it
pass
finally:
@@ -250,7 +249,7 @@ class UDSLogHandler(logging.handlers.RotatingFileHandler):
# Send warning and error messages to systemd journal
if record.levelno >= logging.WARNING:
msg = getMsg(removeLevel=False)
msg = formatMessage(clearLevel=False)
# Send to systemd journaling, transforming identificator and priority
identificator = 'UDS-' + os.path.basename(self.baseFilename).split('.')[0]
# convert syslog level to systemd priority

4
server/src/uds/core/workers/log.py
View File

@@ -44,7 +44,7 @@ logger = logging.getLogger(__name__)
class LogMaintenance(Job):
frecuency = 120 # Once every two minutes
frecuency = 7200 # Once every two hours
# frecuency_cfg = GlobalConfig.XXXX
friendly_name = 'Log maintenance'
@@ -65,7 +65,7 @@ class LogMaintenance(Job):
continue
max_elements = ownerType.get_max_elements()
if 0 < max_elements < count:
if 0 < max_elements < count: # Negative max elements means "unlimited"
# We will delete the oldest ones
for record in models.Log.objects.filter(owner_id=owner_id, owner_type=owner_type).order_by('created', 'id')[: count - max_elements + 1]:
record.delete()

2
server/src/uds/migrations/0044_notification_notifier_servicetokenalias_and_more.py
View File

@@ -35,7 +35,7 @@ def update_network_model(apps, schema_editor): # pylint: disable=unused-argumen
net.version = 4 # Previous versions only supported ipv4
net.save(update_fields=['start', 'end', 'version'])
except Exception as e:
print(f'Error updating network model: {e}')
print(f'Error updating network model: {e}') # Will fail on pytest, but it's ok
class Migration(migrations.Migration):