shaba/openuds
1
0
Fork 1
mirror of https://github.com/dkmstr/openuds.git synced 2025-01-21 18:03:54 +03:00
Code Issues Releases Wiki Activity

More linting fixes

Browse Source
This commit is contained in:
Adolfo Gómez García 2023-04-15 05:33:01 +02:00
parent 54efa40eef
commit 025375e2fe
No known key found for this signature in database
GPG Key ID: DD1ABF20724CDA23
24 changed files with 248 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/.pylintrc
View File

@ -555,7 +555,10 @@ contextmanager-decorators=contextlib.contextmanager
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
generated-members= .*.objects
.*.DoesNotExist.*
.+service,
.+osmanager,
# Tells whether to warn about missing members when the owner of the attribute
# is inferred to be None.

8
server/src/uds/REST/handlers.py
View File

@ -112,9 +112,7 @@ class Handler:
self.needs_admin or self.needs_staff
) and not self.authenticated: # If needs_admin, must also be authenticated
raise Exception(
'class {} is not authenticated but has needs_admin or needs_staff set!!'.format(
self.__class__
)
f'class {self.__class__} is not authenticated but has needs_admin or needs_staff set!!'
)
self._request = request
@ -314,7 +312,7 @@ class Handler:
return net.contains(
GlobalConfig.ADMIN_TRUSTED_SOURCES.get(True), self._request.ip
)
except Exception as e:
except Exception:
logger.warning(
'Error checking truted ADMIN source: "%s" does not seems to be a valid network string. Using Unrestricted access.',
GlobalConfig.ADMIN_TRUSTED_SOURCES.get(),
@ -368,4 +366,4 @@ class Handler:
for name in names:
if name in self._params:
return self._params[name]
return ''
return ''

16
server/src/uds/REST/methods/connection.py
View File

@ -76,7 +76,7 @@ class Connection(Handler):
error = errors.errorString(error)
error = str(error) # Ensure error is an string
if errorCode != 0:
error += ' (code {0:04X})'.format(errorCode)
error += f' (code {errorCode:04X})'
res['error'] = error
res['retryable'] = '1' if retryable else '0'
@ -100,8 +100,8 @@ class Connection(Handler):
(
ip,
userService,
iads,
trans,
_, # iads,
_, #trans,
itrans,
) = userServiceManager().getService( # pylint: disable=unused-variable
self._user,
@ -139,7 +139,7 @@ class Connection(Handler):
(
ip,
userService,
userServiceInstance,
_, # userServiceInstance,
transport,
transportInstance,
) = res # pylint: disable=unused-variable
@ -172,14 +172,14 @@ class Connection(Handler):
logger.exception("Exception")
return Connection.result(error=str(e))
def getTicketContent(self, ticketId: str) -> typing.Dict[str, typing.Any]:
return {} # TODO: use this for something?
def getTicketContent(self, ticketId: str) -> typing.Dict[str, typing.Any]: # pylint: disable=unused-argument
return {}
def getUdsLink(self, idService: str, idTransport: str) -> typing.Dict[str, typing.Any]:
# Returns the UDS link for the user & transport
self._request.user = self._user # type: ignore
self._request._cryptedpass = self._session['REST']['password'] # type: ignore
self._request._scrambler = self._request.META['HTTP_SCRAMBLER'] # type: ignore
setattr(self._request, '_cryptedpass', self._session['REST']['password']) # type: ignore # pylint: disable=protected-access
setattr(self._request, '_scrambler', self._request.META['HTTP_SCRAMBLER']) # type: ignore # pylint: disable=protected-access
linkInfo = services.enableService(
self._request, idService=idService, idTransport=idTransport
)

41
server/src/uds/core/auths/auth.py
View File

@ -94,7 +94,12 @@ def getUDSCookie(
if 'uds' not in request.COOKIES:
cookie = cryptoManager().randomString(UDS_COOKIE_LENGTH)
if response is not None:
response.set_cookie('uds', cookie, samesite='Lax', httponly=GlobalConfig.ENHANCED_SECURITY.getBool())
response.set_cookie(
'uds',
cookie,
samesite='Lax',
httponly=GlobalConfig.ENHANCED_SECURITY.getBool(),
)
request.COOKIES['uds'] = cookie
else:
cookie = request.COOKIES['uds'][:UDS_COOKIE_LENGTH]
@ -164,7 +169,7 @@ def webLoginRequired(
if not request.user or not request.authorized:
return HttpResponseRedirect(reverse('page.login'))
if admin in (True, 'admin'):
if admin in (True, 'admin'):
if request.user.isStaff() is False or (
admin == 'admin' and not request.user.is_admin
):
@ -198,7 +203,7 @@ def trustedSourceRequired(
try:
if not isTrustedSource(request.ip):
return HttpResponseForbidden()
except Exception as e:
except Exception:
logger.warning(
'Error checking trusted source: "%s" does not seems to be a valid network string. Using Unrestricted access.',
GlobalConfig.TRUSTED_SOURCES.get(),
@ -398,7 +403,9 @@ def webLogin(
Helper function to, once the user is authenticated, store the information at the user session.
@return: Always returns True
"""
from uds import REST
from uds import ( # pylint: disable=import-outside-toplevel # to avoid circular imports
REST,
)
if (
user.id != ROOT_ID
@ -421,7 +428,9 @@ def webLogin(
password = '' # nosec: clear password if zero trust is enabled
request.session[USER_KEY] = user.id
request.session[PASS_KEY] = codecs.encode(cryptoManager().symCrypt(password, cookie), "base64").decode() # as str
request.session[PASS_KEY] = codecs.encode(
cryptoManager().symCrypt(password, cookie), "base64"
).decode() # as str
# Ensures that this user will have access through REST api if logged in through web interface
# Note that REST api will set the session expiry to selected value if user is an administrator
@ -451,7 +460,9 @@ def webPassword(request: HttpRequest) -> str:
passkey, getUDSCookie(request)
) # recover as original unicode string
else: # No session, get from _session instead, this is an "client" REST request
return cryptoManager().symDecrpyt(request._cryptedpass, request._scrambler) # type: ignore
return cryptoManager().symDecrpyt(
getattr(request, '_cryptedpass'), getattr(request, '_scrambler')
)
def webLogout(
@ -479,8 +490,6 @@ def webLogout(
)
else: # No user, redirect to /
return HttpResponseRedirect(reverse('page.login'))
except Exception:
raise
finally:
# Try to delete session
request.session.flush()
@ -520,9 +529,7 @@ def authLogLogin(
log.doLog(
authenticator,
level,
'user {} has {} from {} where os is {}'.format(
userName, logStr, request.ip, request.os.os.name
),
f'user {userName} has {logStr} from {request.ip} where os is {request.os.os.name}',
log.WEB,
)
@ -532,12 +539,10 @@ def authLogLogin(
log.doLog(
user,
level,
'{} from {} where OS is {}'.format(
logStr, request.ip, request.os.os.name
),
f'{logStr} from {request.ip} where OS is {request.os.os.name}',
log.WEB,
)
except models.User.DoesNotExist:
except models.User.DoesNotExist: # pylint: disable=no-member
pass
@ -546,9 +551,7 @@ def authLogLogout(request: 'ExtendedHttpRequest') -> None:
log.doLog(
request.user.manager,
log.INFO,
'user {} has logged out from {}'.format(request.user.name, request.ip),
f'user {request.user.name} has logged out from {request.ip}',
log.WEB,
)
log.doLog(
request.user, log.INFO, 'has logged out from {}'.format(request.ip), log.WEB
)
log.doLog(request.user, log.INFO, f'has logged out from {request.ip}', log.WEB)

2
server/src/uds/core/auths/authfactory.py
View File

@ -40,4 +40,4 @@ if typing.TYPE_CHECKING:
class AuthsFactory(factory.ModuleFactory['Authenticator']):
pass
pass

12
server/src/uds/core/auths/exceptions.py
View File

@ -37,24 +37,18 @@ class AuthenticatorException(Exception):
Generic authentication exception
"""
pass
class InvalidUserException(AuthenticatorException):
"""
Invalid user specified. The user cant access the requested service
"""
pass
class InvalidAuthenticatorException(AuthenticatorException):
"""
Invalida authenticator has been specified
"""
pass
class Redirect(AuthenticatorException):
"""
@ -62,20 +56,14 @@ class Redirect(AuthenticatorException):
Used in authUrlCallback to indicate that redirect is needed
"""
pass
class Logout(AuthenticatorException):
"""
This exceptions redirects logouts an user and redirects to an url
"""
pass
class MFAError(AuthenticatorException):
"""
This exceptions indicates than an MFA error has ocurred
"""
pass

20
server/src/uds/core/auths/groups_manager.py
View File

@ -42,6 +42,7 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__)
class _LocalGrp(typing.NamedTuple):
name: str
group: 'Group'
@ -54,6 +55,7 @@ class _LocalGrp(typing.NamedTuple):
"""
return name.casefold() == self.name.casefold()
class GroupsManager:
"""
Manages registered groups for an specific authenticator.
@ -85,7 +87,9 @@ class GroupsManager:
self._dbAuthenticator = dbAuthenticator
# We just get active groups, inactive aren't visible to this class
self._groups = []
if dbAuthenticator.id: # If "fake" authenticator (that is, root user with no authenticator in fact)
if (
dbAuthenticator.id
): # If "fake" authenticator (that is, root user with no authenticator in fact)
for g in dbAuthenticator.groups.filter(state=State.ACTIVE, is_meta=False):
name = g.name.lower()
isPattern = name.find('pat:') == 0 # Is a pattern?
@ -93,7 +97,7 @@ class GroupsManager:
_LocalGrp(
name=name[4:] if isPattern else name,
group=Group(g),
is_pattern=isPattern
is_pattern=isPattern,
)
)
@ -127,7 +131,9 @@ class GroupsManager:
"""
returns the list of valid groups (:py:class:uds.core.auths.group.Group)
"""
from uds.models import Group as DBGroup
from uds.models import ( # pylint: disable=import-outside-toplevel
Group as DBGroup,
)
valid_id_list: typing.List[int] = []
for group in self._groups:
@ -139,7 +145,9 @@ class GroupsManager:
for db_group in DBGroup.objects.filter(
manager__id=self._dbAuthenticator.id, is_meta=True
): # @UndefinedVariable
gn = db_group.groups.filter(id__in=valid_id_list, state=State.ACTIVE).count()
gn = db_group.groups.filter(
id__in=valid_id_list, state=State.ACTIVE
).count()
if db_group.meta_if_any and gn > 0:
gn = db_group.groups.count()
if (
@ -183,7 +191,7 @@ class GroupsManager:
self.validate(n)
else:
for n in self._checkAllGroups(groupName):
self._groups[n] = self._groups[n]._replace(is_valid=True)
self._groups[n] = self._groups[n]._replace(is_valid=True)
def isValid(self, groupName: str) -> bool:
"""
@ -196,4 +204,4 @@ class GroupsManager:
return False
def __str__(self):
return "Groupsmanager: {0}".format(self._groups)
return f'Groupsmanager: {self._groups}'

5
server/src/uds/core/auths/user.py
View File

@ -81,9 +81,9 @@ class User:
:note: Once obtained valid groups, it caches them until object removal.
"""
from uds.models.user import (
from uds.models.user import ( # pylint: disable=import-outside-toplevel
User as DBUser,
) # pylint: disable=redefined-outer-name
)
if self._groups is None:
if self._manager.isExternalSource:
@ -92,7 +92,6 @@ class User:
logger.debug(self._groups)
# This is just for updating "cached" data of this user, we only get real groups at login and at modify user operation
usr = DBUser.objects.get(pk=self._dbUser.id) # @UndefinedVariable
lst: typing.List[int] = []
usr.groups.set((g.dbGroup().id for g in self._groups if g.dbGroup().is_meta is False)) # type: ignore
else:
# From db

2
server/src/uds/core/jobs/__init__.py
View File

@ -45,6 +45,6 @@ def factory() -> 'JobsFactory':
"""
Returns a singleton to a jobs factory
"""
from .jobs_factory import JobsFactory # pylint: disable=redefined-outer-name
from .jobs_factory import JobsFactory # pylint: disable=import-outside-toplevel
return JobsFactory()

2
server/src/uds/core/jobs/delayed_task.py
View File

@ -71,7 +71,7 @@ class DelayedTask(Environmentable):
"""
Utility method that allows to register a Delayedtask
"""
from .delayed_task_runner import DelayedTaskRunner
from .delayed_task_runner import DelayedTaskRunner # pylint: disable=import-outside-toplevel
if check and DelayedTaskRunner.runner().checkExists(tag):
return

6
server/src/uds/core/jobs/jobs_factory.py
View File

@ -47,9 +47,9 @@ class JobsFactory(factory.Factory['Job']):
"""
Ensures that uds core workers are correctly registered in database and in factory
"""
from uds.models import Scheduler, getSqlDatetime
from uds.core.util.state import State
from uds.core import workers
from uds.models import Scheduler, getSqlDatetime # pylint: disable=import-outside-toplevel
from uds.core.util.state import State # pylint: disable=import-outside-toplevel
from uds.core import workers # pylint: disable=import-outside-toplevel
try:
logger.debug('Ensuring that jobs are registered inside database')

4
server/src/uds/core/jobs/scheduler.py
View File

@ -186,8 +186,8 @@ class Scheduler:
# I have got some deadlock errors, but looking at that url, i found that it is not so abnormal
# logger.debug('Deadlock, no problem at all :-) (sounds hards, but really, no problem, will retry later :-) )')
raise DatabaseError(
'Database access problems. Retrying connection ({})'.format(e)
)
f'Database access problems. Retrying connection ({e})'
) from e
@staticmethod
def releaseOwnShedules() -> None:

45
server/src/uds/core/managers/crypto.py
View File

@ -137,7 +137,7 @@ class CryptoManager(metaclass=singleton.Singleton):
modes.CBC(b'udsinitvectoruds'),
backend=default_backend(),
)
rndStr = secrets.token_bytes(16) # Same as block size of CBC (that is 16 here)
rndStr = secrets.token_bytes(16) # Same as block size of CBC (that is 16 here)
paddedLength = ((len(text) + 4 + 15) // 16) * 16
toEncode = (
struct.pack('>i', len(text)) + text + rndStr[: paddedLength - len(text) - 4]
@ -163,11 +163,11 @@ class CryptoManager(metaclass=singleton.Singleton):
toDecode = decryptor.update(text) + decryptor.finalize()
return toDecode[4 : 4 + struct.unpack('>i', toDecode[:4])[0]]
# Fast encription using django SECRET_KEY as key
def fastCrypt(self, data: bytes) -> bytes:
return self.AESCrypt(data, UDSK)
# Fast decryption using django SECRET_KEY as key
def fastDecrypt(self, data: bytes) -> bytes:
return self.AESDecrypt(data, UDSK)
@ -240,8 +240,8 @@ class CryptoManager(metaclass=singleton.Singleton):
# If invalid certificate, will raise an exception
try:
return x509.load_pem_x509_certificate(certificate, default_backend())
except Exception:
raise Exception('Invalid certificate')
except Exception as e:
raise Exception('Invalid certificate') from e
def certificateString(self, certificate: str) -> str:
# Remove -----.*-----\n strings using regex
@ -251,7 +251,6 @@ class CryptoManager(metaclass=singleton.Singleton):
"""
Get a random secret string from config.SECRET_KEY
"""
from django.conf import settings
return settings.SECRET_KEY[:length]
def salt(self, length: int = 16) -> str:
@ -269,22 +268,28 @@ class CryptoManager(metaclass=singleton.Singleton):
return '{SHA256SALT}' + salt + str(hashlib.sha3_256(value).hexdigest())
def checkHash(self, value: typing.Union[str, bytes], hash: str) -> bool:
def checkHash(self, value: typing.Union[str, bytes], hashValue: str) -> bool:
if isinstance(value, str):
value = value.encode()
if not value:
return not hash
return not hashValue
if hash[:8] == '{SHA256}':
return secrets.compare_digest(hashlib.sha3_256(value).hexdigest(), hash[8:])
elif hash[:12] == '{SHA256SALT}':
if hashValue[:8] == '{SHA256}':
return secrets.compare_digest(
hashlib.sha3_256(value).hexdigest(), hashValue[8:]
)
elif hashValue[:12] == '{SHA256SALT}':
# Extract 16 chars salt and hash
salt = hash[12:28].encode()
salt = hashValue[12:28].encode()
value = salt + value
return secrets.compare_digest(hashlib.sha3_256(value).hexdigest(), hash[28:])
return secrets.compare_digest(
hashlib.sha3_256(value).hexdigest(), hashValue[28:]
)
else: # Old sha1
return secrets.compare_digest(hash, str(hashlib.sha1(value).hexdigest())) # nosec: Old compatibility SHA1, not used anymore but need to be supported
return secrets.compare_digest(
hashValue, str(hashlib.sha1(value).hexdigest()) # nosec: Old compatibility SHA1, not used anymore but need to be supported
) # nosec: Old compatibility SHA1, not used anymore but need to be supported
def uuid(self, obj: typing.Any = None) -> str:
"""
@ -297,7 +302,7 @@ class CryptoManager(metaclass=singleton.Singleton):
elif isinstance(obj, bytes):
obj = obj.decode('utf8') # To binary
else:
obj = '{}'.format(obj)
obj = str(obj)
return str(
uuid.uuid5(self._namespace, obj)
@ -309,8 +314,8 @@ class CryptoManager(metaclass=singleton.Singleton):
def unique(self) -> str:
return hashlib.sha3_256(
(
self.randomString(24, True)
+ datetime.datetime.now().strftime('%H%M%S%f')
).encode()
).hexdigest()
(
self.randomString(24, True)
+ datetime.datetime.now().strftime('%H%M%S%f')
).encode()
).hexdigest()

27
server/src/uds/core/managers/notifications.py
View File

@ -32,8 +32,6 @@
import logging
import typing
from django.db import transaction
from uds.core.util import singleton
from uds.models.notifications import Notification, NotificationLevel
@ -42,19 +40,32 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__)
class NotificationsManager(metaclass=singleton.Singleton):
"""
This class manages alerts and notifications
"""
def __init__(self):
pass
@staticmethod
def manager() -> 'NotificationsManager':
return NotificationsManager() # Singleton pattern will return always the same instance
return (
NotificationsManager()
) # Singleton pattern will return always the same instance
def notify(self, group: str, identificator: str, level: NotificationLevel, message: str, *args) -> None:
logger.debug('Notify: %s, %s, %s, %s, [%s]', group, identificator, level, message, args)
def notify(
self,
group: str,
identificator: str,
level: NotificationLevel,
message: str,
*args
) -> None:
logger.debug(
'Notify: %s, %s, %s, %s, [%s]', group, identificator, level, message, args
)
# Format the string
try:
message = message % args
@ -63,7 +74,9 @@ class NotificationsManager(metaclass=singleton.Singleton):
# Store the notification on local persistent storage
# Will be processed by UDS backend
with Notification.atomicPersistent():
notify = Notification(group=group, identificator=identificator, level=level, message=message)
notify = Notification(
group=group, identificator=identificator, level=level, message=message
)
Notification.savePersistent(notify)
def registerGroup(self, group: str) -> None:
@ -71,11 +84,9 @@ class NotificationsManager(metaclass=singleton.Singleton):
Registers a new group.
This is used to group notifications
"""
pass
def registerIdentificator(self, group: str, identificator: str) -> None:
"""
Registers a new identificator.
This is used to identify notifications
"""
pass

19
server/src/uds/core/managers/publication.py
View File

@ -33,10 +33,10 @@ import typing
import logging
import datetime
from uds.core.util.serializer import serialize
from django.utils.translation import gettext as _
from django.db import transaction
from uds.core.util.serializer import serialize
from uds.core.jobs.delayed_task import DelayedTask
from uds.core.jobs.delayed_task_runner import DelayedTaskRunner
from uds.core.util.config import GlobalConfig
@ -81,9 +81,10 @@ class PublicationOldMachinesCleaner(DelayedTask):
in_use=False, state_date=now
)
servicePoolPub.deployed_service.markOldUserServicesAsRemovables(activePub)
except Exception: # nosec: Removed publication, no problem at all, just continue
except (
Exception
): # nosec: Removed publication, no problem at all, just continue
pass
class PublicationLauncher(DelayedTask):
@ -127,7 +128,9 @@ class PublicationLauncher(DelayedTask):
)
servicePool.save()
PublicationFinishChecker.checkAndUpdateState(servicePoolPub, pi, state)
except ServicePoolPublication.DoesNotExist: # Deployed service publication has been removed from database, this is ok, just ignore it
except (
ServicePoolPublication.DoesNotExist
): # Deployed service publication has been removed from database, this is ok, just ignore it
pass
except Exception:
logger.exception("Exception launching publication")
@ -321,7 +324,7 @@ class PublicationManager(metaclass=singleton.Singleton):
publication.delete()
except Exception:
logger.info('Could not delete %s', publication)
raise PublishException(str(e))
raise PublishException(str(e)) from e
def cancel(
self, publication: ServicePoolPublication
@ -363,7 +366,7 @@ class PublicationManager(metaclass=singleton.Singleton):
)
return publication
except Exception as e:
raise PublishException(str(e))
raise PublishException(str(e)) from e
def unpublish(
self, servicePoolPub: ServicePoolPublication
@ -389,4 +392,4 @@ class PublicationManager(metaclass=singleton.Singleton):
servicePoolPub, pubInstance, state
)
except Exception as e:
raise PublishException(str(e))
raise PublishException(str(e)) from e

1
server/src/uds/core/managers/stats.py
View File

@ -239,7 +239,6 @@ class StatsManager(metaclass=singleton.Singleton):
# Append to numpy array
yield last
stamp += intervalType.seconds()
def cleanupCounters(self):
"""

7
server/src/uds/core/managers/task.py
View File

@ -75,14 +75,13 @@ class TaskManager(metaclass=singleton.Singleton):
def __init__(self):
self.keepRunning = True
self.threads = []
pass
@staticmethod
def manager() -> 'TaskManager':
return TaskManager()
@staticmethod
def sigTerm(sigNum, frame):
def sigTerm(sigNum, frame): # pylint: disable=unused-argument
"""
This method will ensure that we finish correctly current running task before exiting.
If we need to stop cause something went wrong (that should not happen), we must send sigterm, wait a while (10-20 secs) and after that send sigkill
@ -102,12 +101,12 @@ class TaskManager(metaclass=singleton.Singleton):
logger.info("Registering sheduled tasks")
# Simply import this to make workers "auto import themself"
from uds.core import workers # @UnusedImport pylint: disable=unused-import
from uds.core import workers # pylint: disable=unused-import, import-outside-toplevel
def addOtherTasks(self) -> None:
logger.info("Registering other tasks")
from uds.core.messaging.processor import MessageProcessorThread
from uds.core.messaging.processor import MessageProcessorThread # pylint: disable=import-outside-toplevel
thread = MessageProcessorThread()
thread.start()

2
server/src/uds/core/managers/user_preferences.py
View File

@ -35,8 +35,6 @@
import logging
import typing
from django.utils.translation import gettext as _
logger = logging.getLogger(__name__)
class CommonPrefs:

69
server/src/uds/core/managers/user_service.py
View File

@ -39,7 +39,6 @@ from django.utils.translation import gettext as _
from django.db.models import Q
from django.db import transaction
from uds.core.services.exceptions import OperationException
from uds.core.util.config import GlobalConfig
from uds.core.util.state import State
from uds.core.util import log
from uds.core.services.exceptions import (
@ -58,7 +57,6 @@ from uds.models import (
User,
ServicePoolPublication,
)
from uds.models.meta_pool import MetaPoolMember
from uds.core import services, transports
from uds.core.util import singleton
from uds.core.util.stats import events
@ -74,6 +72,7 @@ logger = logging.getLogger(__name__)
traceLogger = logging.getLogger('traceLog')
operationsLogger = logging.getLogger('operationsLog')
class UserServiceManager(metaclass=singleton.Singleton):
def __init__(self):
pass
@ -89,9 +88,11 @@ class UserServiceManager(metaclass=singleton.Singleton):
@staticmethod
def getStateFilter(service: 'models.Service') -> Q:
if service.oldMaxAccountingMethod: # If no limits and accounting method is not old one
if (
service.oldMaxAccountingMethod
): # If no limits and accounting method is not old one
# Valid states are: PREPARING, USABLE
states = [State.PREPARING, State.USABLE]
states = [State.PREPARING, State.USABLE]
else: # New accounting method selected
states = [State.PREPARING, State.USABLE, State.REMOVING, State.REMOVABLE]
return Q(state__in=states)
@ -126,7 +127,9 @@ class UserServiceManager(metaclass=singleton.Singleton):
return False
if self.getExistingUserServices(service) >= (serviceInstance.maxDeployed or 1):
operationsLogger.info('Maximum number of user services reached for service: {}'.format(service.name))
operationsLogger.info(
'Maximum number of user services reached for service: %s', service.name
)
return True
return False
@ -236,12 +239,14 @@ class UserServiceManager(metaclass=singleton.Singleton):
)
else:
raise Exception(
'Invalid publication creating service assignation: {} {}'.format(
servicePool, user
)
f'Invalid publication creating service assignation: {servicePool.name} {user.pretty_name}'
)
else:
operationsLogger.info('Creating a new assigned element for user %s on pool %s', user.pretty_name, servicePool.name)
operationsLogger.info(
'Creating a new assigned element for user %s on pool %s',
user.pretty_name,
servicePool.name,
)
assigned = self._createAssignedAtDbForNoPublication(servicePool, user)
assignedInstance = assigned.getInstance()
@ -274,9 +279,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
)
else:
raise Exception(
'Invalid publication creating service assignation: {} {}'.format(
servicePool, user
)
f'Invalid publication creating service assignation: {servicePool.name} {user.pretty_name}'
)
else:
operationsLogger.info(
@ -562,7 +565,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
log.doLog(
servicePool,
log.WARN,
'Max number of services reached: {}'.format(servicePool.max_srvs),
f'Max number of services reached: {servicePool.max_srvs}',
log.INTERNAL,
)
raise MaxServicesReachedError()
@ -591,8 +594,8 @@ class UserServiceManager(metaclass=singleton.Singleton):
)
serviceInstance = servicePool.service.getInstance()
if (
serviceInstance.isAvailable() and
removing >= serviceInstance.parent().getMaxRemovingServices()
serviceInstance.isAvailable()
and removing >= serviceInstance.parent().getMaxRemovingServices()
and serviceInstance.parent().getIgnoreLimits() is False
):
return False
@ -626,7 +629,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
try:
state = userServiceInstance.setReady()
except Exception as e:
logger.warn('Could not check readyness of %s: %s', userService, e)
logger.warning('Could not check readyness of %s: %s', userService, e)
return False
logger.debug('State: %s', state)
@ -823,8 +826,8 @@ class UserServiceManager(metaclass=singleton.Singleton):
try:
transport: Transport = Transport.objects.get(uuid=idTransport)
except Exception:
raise InvalidServiceException()
except Exception as e:
raise InvalidServiceException() from e
# Ensures that the transport is allowed for this service
if userService.deployed_service.transports.filter(id=transport.id).count() == 0:
@ -852,7 +855,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
log.doLog(
userService,
log.INFO,
"User {0} from {1} has initiated access".format(user.name, srcIp),
f"User {user.pretty_name} from {srcIp} has initiated access",
log.WEB,
)
# If ready, show transport for this service, if also ready ofc
@ -868,9 +871,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
log.doLog(
userService,
log.WARN,
"User service is not accessible due to invalid UUID (ip {0})".format(
ip
),
f'User service is not accessible due to invalid UUID (user: {user.pretty_name}, ip: {ip})',
log.TRANSPORT,
)
logger.debug('UUID check failed for user service %s', userService)
@ -924,9 +925,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
log.doLog(
userService,
log.WARN,
"User {} from {} tried to access, but service was not ready".format(
user.name, srcIp
),
f'User {user.pretty_name} from {srcIp} tried to access, but service was not ready',
log.WEB,
)
@ -1007,13 +1006,21 @@ class UserServiceManager(metaclass=singleton.Singleton):
sortPools = [(p.pool.usage(), p.pool) for p in poolMembers]
else:
sortPools = [
(random.randint(0, 10000), p.pool) for p in poolMembers # nosec: just a suffle, not a crypto (to get a round robin-like behavior)
(
random.randint(
0, 10000
), # nosec: just a suffle, not a crypto (to get a round robin-like behavior)
p.pool,
)
for p in poolMembers
] # Just shuffle them
# Sort pools related to policy now, and xtract only pools, not sort keys
# split resuult in two lists, 100% full and not 100% full
# Remove "full" pools (100%) from result and pools in maintenance mode, not ready pools, etc...
sortedPools = sorted(sortPools, key=operator.itemgetter(0)) # sort by priority (first element)
sortedPools = sorted(
sortPools, key=operator.itemgetter(0)
) # sort by priority (first element)
pools: typing.List[ServicePool] = [
p[1] for p in sortedPools if p[1].usage() < 100 and p[1].isUsable()
]
@ -1064,7 +1071,9 @@ class UserServiceManager(metaclass=singleton.Singleton):
# If already assigned, and HA is enabled, check if it is accessible
if meta.ha_policy == MetaPool.HA_POLICY_ENABLED:
# Check that servide is accessible
if not alreadyAssigned.deployed_service.service.getInstance().isAvailable(): # Not available, mark for removal
if (
not alreadyAssigned.deployed_service.service.getInstance().isAvailable()
): # Not available, mark for removal
alreadyAssigned.release()
raise Exception() # And process a new access
@ -1118,9 +1127,7 @@ class UserServiceManager(metaclass=singleton.Singleton):
log.doLog(
meta,
log.WARN,
"No user service accessible from device (ip {}, os: {})".format(
srcIp, os.os.name
),
f'No user service accessible from device (ip {srcIp}, os: {os.os.name})',
log.SERVICE,
)
raise InvalidServiceException(

58
server/src/uds/core/managers/userservice/comms.py
View File

@ -69,7 +69,7 @@ def _requestActor(
if not url:
# logger.warning('No notification is made because agent does not supports notifications: %s', userService.friendly_name)
raise NoActorComms(
'No notification urls for {}'.format(userService.friendly_name)
f'No notification urls for {userService.friendly_name}'
)
minVersion = minVersion or '3.5.0'
@ -79,42 +79,38 @@ def _requestActor(
'Pool %s has old actors (%s)', userService.deployed_service.name, version
)
raise OldActorVersion(
'Old actor version {} for {}'.format(version, userService.friendly_name)
f'Old actor version {version} for {userService.friendly_name}'.format(version, userService.friendly_name)
)
url += '/' + method
proxy = userService.deployed_service.proxy
try:
if proxy:
r = proxy.doProxyRequest(url=url, data=data, timeout=TIMEOUT)
verify: typing.Union[bool, str]
cert = userService.getProperty('cert') or ''
# cert = '' # Uncomment to test without cert
if cert:
# Generate temp file, and delete it after
with tempfile.NamedTemporaryFile('wb', delete=False) as f:
f.write(cert.encode()) # Save cert
verify = f.name
else:
verify: typing.Union[bool, str]
cert = userService.getProperty('cert') or ''
# cert = '' # Uncomment to test without cert
if cert:
# Generate temp file, and delete it after
verify = tempfile.mktemp('udscrt')
with open(verify, 'wb') as f:
f.write(cert.encode()) # Save cert
else:
verify = False
session = secureRequestsSession(verify=cert)
if data is None:
r = session.get(url, verify=verify, timeout=TIMEOUT)
else:
r = session.post(
url,
data=json.dumps(data),
headers={'content-type': 'application/json'},
verify=verify,
timeout=TIMEOUT,
)
if verify:
try:
os.remove(typing.cast(str, verify))
except Exception:
logger.exception('removing verify')
verify = False
session = secureRequestsSession(verify=cert)
if data is None:
r = session.get(url, verify=verify, timeout=TIMEOUT)
else:
r = session.post(
url,
data=json.dumps(data),
headers={'content-type': 'application/json'},
verify=verify,
timeout=TIMEOUT,
)
if verify:
try:
os.remove(typing.cast(str, verify))
except Exception:
logger.exception('removing verify')
js = r.json()
if version >= '3.0.0':

16
server/src/uds/core/managers/userservice/opchecker.py
View File

@ -101,7 +101,7 @@ class StateUpdater:
try:
executor()
except Exception as e:
self.setError('Exception: {}'.format(e))
self.setError(f'Exception: {e}')
logger.debug('Executor for %s done', self.userService.friendly_name)
@ -196,16 +196,12 @@ class UpdateFromCanceling(StateUpdater):
class UpdateFromOther(StateUpdater):
def finish(self):
self.setError(
'Unknown running transition from {}'.format(
State.toString(self.userService.state)
)
f'Unknown running transition from {State.toString(self.userService.state)}'
)
def running(self):
self.setError(
'Unknown running transition from {}'.format(
State.toString(self.userService.state)
)
f'Unknown running transition from {State.toString(self.userService.state)}'
)
@ -267,7 +263,7 @@ class UserServiceOpChecker(DelayedTask):
except Exception as e:
logger.exception('Checking service state')
log.doLog(userService, log.ERROR, 'Exception: {}'.format(e), log.INTERNAL)
log.doLog(userService, log.ERROR, f'Exception: {e}', log.INTERNAL)
userService.setState(State.ERROR)
userService.save(update_fields=['data'])
@ -300,7 +296,7 @@ class UserServiceOpChecker(DelayedTask):
logger.debug("uService instance class: %s", ci.__class__)
state = ci.checkState()
UserServiceOpChecker.checkAndUpdateState(uService, ci, state)
except UserService.DoesNotExist as e:
except UserService.DoesNotExist as e: # pylint: disable=no-member
logger.error(
'User service not found (erased from database?) %s : %s', e.__class__, e
)
@ -308,7 +304,7 @@ class UserServiceOpChecker(DelayedTask):
# Exception caught, mark service as errored
logger.exception("Error %s, %s :", e.__class__, e)
if uService:
log.doLog(uService, log.ERROR, 'Exception: {}'.format(e), log.INTERNAL)
log.doLog(uService, log.ERROR, f'Exception: {e}', log.INTERNAL)
try:
uService.setState(State.ERROR)
uService.save(update_fields=['data'])

10
server/src/uds/core/util/request.py
View File

@ -46,16 +46,6 @@ class ExtendedHttpRequest(HttpRequest):
os: 'DetectedOsInfo'
user: typing.Optional[User]
authorized: bool
def __init__(self) -> None:
super().__init__()
class ExtendedHttpRequestWithUser(ExtendedHttpRequest):
user: User
def __init__(self) -> None:
super().__init__()

71
server/src/uds/models/service_pool.py
View File

@ -71,6 +71,7 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__)
# pylint: disable=too-many-public-methods
class ServicePool(UUIDModel, TaggingMixin): # type: ignore
"""
@ -92,11 +93,15 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
related_name='deployedServices',
on_delete=models.CASCADE,
)
transports: 'models.ManyToManyField[Transport, ServicePool]' = models.ManyToManyField(
Transport, related_name='deployedServices', db_table='uds__ds_trans'
transports: 'models.ManyToManyField[Transport, ServicePool]' = (
models.ManyToManyField(
Transport, related_name='deployedServices', db_table='uds__ds_trans'
)
)
assignedGroups: 'models.ManyToManyField[Group, ServicePool]' = models.ManyToManyField(
Group, related_name='deployedServices', db_table='uds__ds_grps'
assignedGroups: 'models.ManyToManyField[Group, ServicePool]' = (
models.ManyToManyField(
Group, related_name='deployedServices', db_table='uds__ds_grps'
)
)
state = models.CharField(
max_length=1, default=states.servicePool.ACTIVE, db_index=True
@ -200,11 +205,13 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
@staticmethod
def getRestrainedsQuerySet() -> 'models.QuerySet[ServicePool]':
from uds.models.user_service import (
from uds.models.user_service import ( # pylint: disable=import-outside-toplevel
UserService,
) # pylint: disable=redefined-outer-name
from uds.core.util.config import GlobalConfig
from django.db.models import Count
)
from uds.core.util.config import ( # pylint: disable=import-outside-toplevel
GlobalConfig,
)
from django.db.models import Count # pylint: disable=import-outside-toplevel
if GlobalConfig.RESTRAINT_TIME.getInt() <= 0:
return (
@ -260,7 +267,9 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
The time that a service is in restrain mode is 20 minutes by default (1200 secs), but it can be modified
at globalconfig variables
"""
from uds.core.util.config import GlobalConfig
from uds.core.util.config import ( # pylint: disable=import-outside-toplevel
GlobalConfig,
)
if GlobalConfig.RESTRAINT_TIME.getInt() <= 0:
return False # Do not perform any restraint check if we set the globalconfig to 0 (or less)
@ -315,7 +324,7 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
ret = self.recoverValue('toBeReplacedIn')
if ret:
return serializer.deserialize(ret)
except Exception: # nosec: We don't want to fail if there is any exception
# logger.exception('Recovering publication death line')
pass
@ -364,11 +373,15 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
ac.access == states.action.ALLOW
and self.fallbackAccess == states.action.DENY
):
nextE = calendar.CalendarChecker(ac.calendar).nextEvent(chkDateTime, False)
nextE = calendar.CalendarChecker(ac.calendar).nextEvent(
chkDateTime, False
)
if not deadLine or (nextE and deadLine > nextE):
deadLine = nextE
elif ac.access == states.action.DENY: # DENY
nextE = calendar.CalendarChecker(ac.calendar).nextEvent(chkDateTime, True)
nextE = calendar.CalendarChecker(ac.calendar).nextEvent(
chkDateTime, True
)
if not deadLine or (nextE and deadLine > nextE):
deadLine = nextE
@ -482,9 +495,11 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
Ensures that at least a group of groups (database groups) has access to this Service Pool
raise an InvalidUserException if fails check
"""
from uds.core import auths
from uds.core import auths # pylint: disable=import-outside-toplevel
if not set(groups) & set(self.assignedGroups.all()):
if not set(groups) & set(
self.assignedGroups.all() # pylint: disable=no-member
): # pylint: disable=no-member
raise auths.exceptions.InvalidUserException()
def validatePublication(self) -> None:
@ -500,7 +515,10 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
raise InvalidServiceException()
def validateTransport(self, transport) -> None:
if self.transports.filter(id=transport.id).count() == 0:
if (
self.transports.filter(id=transport.id).count() # pylint: disable=no-member
== 0 # pylint: disable=no-member
): # pylint: disable=no-member
raise InvalidServiceException()
def validateUser(self, user: 'User') -> None:
@ -536,7 +554,7 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
Returns:
List of accesible deployed services
"""
from uds.core import services
from uds.core import services # pylint: disable=import-outside-toplevel
servicesNotNeedingPub = [
t.type() for t in services.factory().servicesThatDoNotNeedPublication()
@ -608,7 +626,9 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
No check is done, it simply redirects the request to PublicationManager, where checks are done.
"""
from uds.core.managers import publicationManager
from uds.core.managers import ( # pylint: disable=import-outside-toplevel
publicationManager,
)
publicationManager().publish(self, changeLog)
@ -683,7 +703,7 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
log.doLog(self, level, message, log.INTERNAL)
@staticmethod
def beforeDelete(sender, **kwargs) -> None:
def beforeDelete(sender, **kwargs) -> None: # pylint: disable=unused-argument
"""
Used to invoke the Service class "Destroy" before deleting it from database.
@ -692,7 +712,9 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
:note: If destroy raises an exception, the deletion is not taken.
"""
from uds.core.util.permissions import clean
from uds.core.util.permissions import ( # pylint: disable=import-outside-toplevel
clean,
)
toDelete: 'ServicePool' = kwargs['instance']
@ -735,13 +757,10 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
)
def __str__(self):
return 'Service pool {}({}) with {} as initial, {} as L1 cache, {} as L2 cache, {} as max'.format(
self.name,
self.id,
self.initial_srvs,
self.cache_l1_srvs,
self.cache_l2_srvs,
self.max_srvs,
return (
f'Service pool {self.name}({self.id}) with {self.initial_srvs}'
f' as initial, {self.cache_l1_srvs} as L1 cache, {self.cache_l2_srvs}'
f' as L2 cache, {self.max_srvs} as max'
)

35
server/src/uds/models/user_service.py
View File

@ -52,8 +52,6 @@ if typing.TYPE_CHECKING:
from uds.core import services
from uds.models import (
OSManager,
ServicePool,
ServicePoolPublication,
UserServiceProperty,
UserServiceSession,
AccountUsage,
@ -137,7 +135,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
"""
Simple accessor to deployed service name plus unique name
"""
return "{}\\{}".format(self.deployed_service.name, self.friendly_name)
return f'{self.deployed_service.name}\\{self.friendly_name}'
@property
def destroy_after(self) -> bool:
@ -173,7 +171,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
(see related classes uds.core.util.unique_name_generator and uds.core.util.unique_mac_generator)
"""
return Environment.getEnvForTableElement(
self._meta.verbose_name, # type: ignore
self._meta.verbose_name, # type: ignore # pylint: disable=no-member
self.id,
{
'mac': unique.UniqueMacGenerator,
@ -221,9 +219,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
)
if serviceInstance.deployedType is None:
raise Exception(
'Class {0} needs deployedType but it is not defined!!!'.format(
serviceInstance.__class__.__name__
)
f'Class {serviceInstance.__class__.__name__} needs deployedType but it is not defined!!!'
)
us = serviceInstance.deployedType(
self.getEnvironment(),
@ -440,7 +436,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
:note: If the state is Fase (set to not in use), a check for removal of this deployed service is launched.
"""
from uds.core.managers import userServiceManager
from uds.core.managers import userServiceManager # pylint: disable=import-outside-toplevel
self.in_use = inUse
self.in_use_date = getSqlDatetime()
@ -498,8 +494,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
session.close()
except Exception: # Does not exists, log it and ignore it
logger.warning(
'Session %s does not exists for user deployed service %s'
% (sessionId, self.id)
'Session %s does not exists for user deployed service', self.id
)
def isUsable(self) -> bool:
@ -519,7 +514,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
Returns if this service is ready (not preparing or marked for removal)
"""
# Call to isReady of the instance
from uds.core.managers import userServiceManager
from uds.core.managers import userServiceManager # pylint: disable=import-outside-toplevel
return userServiceManager().isReady(self)
@ -542,7 +537,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
"""
Asks the UserServiceManager to cancel the current operation of this user deployed service.
"""
from uds.core.managers import userServiceManager
from uds.core.managers import userServiceManager # pylint: disable=import-outside-toplevel
userServiceManager().cancel(self)
@ -568,7 +563,7 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
Args:
cacheLevel: New cache level to put object in
"""
from uds.core.managers import userServiceManager
from uds.core.managers import userServiceManager # pylint: disable=import-outside-toplevel
userServiceManager().moveToLevel(self, cacheLevel)
@ -634,18 +629,14 @@ class UserService(UUIDModel): # pylint: disable=too-many-public-methods
return self.deployed_service.testServer(host, port, timeout)
def __str__(self):
return "User service {}, unique_id {}, cache_level {}, user {}, name {}, state {}:{}".format(
self.name,
self.unique_id,
self.cache_level,
self.user,
self.friendly_name,
State.toString(self.state),
State.toString(self.os_state),
return (
f'User service {self.name}, unique_id {self.unique_id},'
f' cache_level {self.cache_level}, user {self.user},'
f' name {self.friendly_name}, state {State.toString(self.state)}:{State.toString(self.os_state)}'
)
@staticmethod
def beforeDelete(sender, **kwargs) -> None:
def beforeDelete(sender, **kwargs) -> None: # pylint: disable=unused-argument
"""
Used to invoke the Service class "Destroy" before deleting it from database.