diff --git a/server/src/uds/core/managers/__init__.py b/server/src/uds/core/managers/__init__.py index 25070f5db..c9f609cfb 100644 --- a/server/src/uds/core/managers/__init__.py +++ b/server/src/uds/core/managers/__init__.py @@ -40,7 +40,6 @@ if typing.TYPE_CHECKING: from .task import TaskManager from .downloads import DownloadsManager from .log import LogManager - from .stats import StatsManager from .user_service import UserServiceManager from .publication import PublicationManager @@ -64,11 +63,6 @@ def logManager() -> 'LogManager': return LogManager.manager() -def statsManager() -> 'StatsManager': - from .stats import StatsManager # pylint: disable=redefined-outer-name - return StatsManager.manager() - - def userServiceManager() -> 'UserServiceManager': from .user_service import UserServiceManager # pylint: disable=redefined-outer-name return UserServiceManager.manager() diff --git a/server/src/uds/core/util/cache.py b/server/src/uds/core/util/cache.py index a228601e6..28fddc305 100644 --- a/server/src/uds/core/util/cache.py +++ b/server/src/uds/core/util/cache.py @@ -129,7 +129,11 @@ class Cache: now = getSqlDatetime() try: DBCache.objects.create( - owner=self._owner, key=key, value=strValue, created=now, validity=validity + owner=self._owner, + key=key, + value=strValue, + created=now, + validity=validity, ) # @UndefinedVariable except Exception: try: diff --git a/server/src/uds/core/util/certs.py b/server/src/uds/core/util/certs.py index 97eae21d1..5f2cd0630 100644 --- a/server/src/uds/core/util/certs.py +++ b/server/src/uds/core/util/certs.py @@ -11,12 +11,13 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa + def selfSignedCert(ip: str) -> typing.Tuple[str, str, str]: key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048, - backend=default_backend(), - ) + public_exponent=65537, + key_size=2048, + backend=default_backend(), + ) # Create a random password for private key password = secrets.token_urlsafe(32) @@ -27,23 +28,25 @@ def selfSignedCert(ip: str) -> typing.Tuple[str, str, str]: now = datetime.utcnow() cert = ( x509.CertificateBuilder() - .subject_name(name) - .issuer_name(name) # self signed, its Issuer DN must match its Subject DN. - .public_key(key.public_key()) - .serial_number(random.SystemRandom().randint(0, 1<<64)) - .not_valid_before(now) - .not_valid_after(now + timedelta(days=10*365)) - .add_extension(basic_contraints, False) - .add_extension(san, False) - .sign(key, hashes.SHA256(), default_backend()) + .subject_name(name) + .issuer_name(name) # self signed, its Issuer DN must match its Subject DN. + .public_key(key.public_key()) + .serial_number(random.SystemRandom().randint(0, 1 << 64)) + .not_valid_before(now) + .not_valid_after(now + timedelta(days=10 * 365)) + .add_extension(basic_contraints, False) + .add_extension(san, False) + .sign(key, hashes.SHA256(), default_backend()) ) - + return ( key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.BestAvailableEncryption(password.encode()) + encryption_algorithm=serialization.BestAvailableEncryption( + password.encode() + ), ).decode(), cert.public_bytes(encoding=serialization.Encoding.PEM).decode(), - password + password, ) diff --git a/server/src/uds/core/util/stats/counters.py b/server/src/uds/core/util/stats/counters.py index cb24322c8..e644da9ea 100644 --- a/server/src/uds/core/util/stats/counters.py +++ b/server/src/uds/core/util/stats/counters.py @@ -36,18 +36,27 @@ import typing from django.utils.translation import ugettext_lazy as _ -from uds.core.managers import statsManager -from uds.models import NEVER, Provider, Service, ServicePool, Authenticator +from uds.core.managers.stats import StatsManager +from uds.models import NEVER, Provider, Service, ServicePool, Authenticator logger = logging.getLogger(__name__) -CounterClass = typing.TypeVar('CounterClass', Provider, Service, ServicePool, Authenticator) +CounterClass = typing.TypeVar( + 'CounterClass', Provider, Service, ServicePool, Authenticator +) # Posible counters, note that not all are used by every posible type # FIRST_COUNTER_TYPE, LAST_COUNTER_TYPE are just a placeholder for sanity checks ( - CT_LOAD, CT_STORAGE, CT_ASSIGNED, CT_INUSE, CT_AUTH_USERS, CT_AUTH_USERS_WITH_SERVICES, CT_AUTH_SERVICES, CT_CACHED + CT_LOAD, + CT_STORAGE, + CT_ASSIGNED, + CT_INUSE, + CT_AUTH_USERS, + CT_AUTH_USERS_WITH_SERVICES, + CT_AUTH_SERVICES, + CT_CACHED, ) = range(8) __caRead: typing.Dict = {} @@ -56,7 +65,12 @@ __transDict: typing.Dict = {} __typeTitles: typing.Dict = {} -def addCounter(obj: CounterClass, counterType: int, counterValue: int, stamp: typing.Optional[datetime.datetime] = None) -> bool: +def addCounter( + obj: CounterClass, + counterType: int, + counterValue: int, + stamp: typing.Optional[datetime.datetime] = None, +) -> bool: """ Adds a counter stat to specified object @@ -68,13 +82,22 @@ def addCounter(obj: CounterClass, counterType: int, counterValue: int, stamp: ty """ type_ = type(obj) if type_ not in __caWrite.get(counterType, ()): # pylint: disable - logger.error('Type %s does not accepts counter of type %s', type_, counterValue, exc_info=True) + logger.error( + 'Type %s does not accepts counter of type %s', + type_, + counterValue, + exc_info=True, + ) return False - return statsManager().addCounter(__transDict[type(obj)], obj.id, counterType, counterValue, stamp) + return StatsManager.manager().addCounter( + __transDict[type(obj)], obj.id, counterType, counterValue, stamp + ) -def getCounters(obj: CounterClass, counterType: int, **kwargs) -> typing.Generator[typing.Tuple[datetime.datetime, int], None, None]: +def getCounters( + obj: CounterClass, counterType: int, **kwargs +) -> typing.Generator[typing.Tuple[datetime.datetime, int], None, None]: """ Get counters @@ -112,7 +135,17 @@ def getCounters(obj: CounterClass, counterType: int, **kwargs) -> typing.Generat else: owner_ids = None - for i in statsManager().getCounters(__transDict[type(obj)], counterType, owner_ids, since, to, kwargs.get('interval'), kwargs.get('max_intervals'), limit, use_max): + for i in StatsManager.manager().getCounters( + __transDict[type(obj)], + counterType, + owner_ids, + since, + to, + kwargs.get('interval'), + kwargs.get('max_intervals'), + limit, + use_max, + ): yield (datetime.datetime.fromtimestamp(i.stamp), i.value) @@ -128,67 +161,67 @@ def _initializeData() -> None: Hides data from global var space """ - __caWrite.update({ - CT_LOAD: (Provider,), - CT_STORAGE: (Service,), - CT_ASSIGNED: (ServicePool,), - CT_INUSE: (ServicePool,), - CT_AUTH_USERS: (Authenticator,), - CT_AUTH_SERVICES: (Authenticator,), - CT_AUTH_USERS_WITH_SERVICES: (Authenticator,), - CT_CACHED: (ServicePool,), - }) + __caWrite.update( + { + CT_LOAD: (Provider,), + CT_STORAGE: (Service,), + CT_ASSIGNED: (ServicePool,), + CT_INUSE: (ServicePool,), + CT_AUTH_USERS: (Authenticator,), + CT_AUTH_SERVICES: (Authenticator,), + CT_AUTH_USERS_WITH_SERVICES: (Authenticator,), + CT_CACHED: (ServicePool,), + } + ) # OBtain ids from variups type of object to retrieve stats def get_Id(obj): return obj.id - def get_P_S_Ids(provider): - return (i.id for i in provider.services.all()) + def get_P_S_Ids(provider) -> typing.Tuple: + return tuple(i.id for i in provider.services.all()) - def get_S_DS_Ids(service): - return (i.id for i in service.deployedServices.all()) + def get_S_DS_Ids(service) -> typing.Tuple: + return tuple(i.id for i in service.deployedServices.all()) - def get_P_S_DS_Ids(provider): - res = () + def get_P_S_DS_Ids(provider) -> typing.Tuple: + res: typing.Tuple = () for i in provider.services.all(): res += get_S_DS_Ids(i) return res - __caRead.update({ - Provider: { - CT_LOAD: get_Id, - CT_STORAGE: get_P_S_Ids, - CT_ASSIGNED: get_P_S_DS_Ids, - CT_INUSE: get_P_S_DS_Ids - }, - Service: { - CT_STORAGE: get_Id, - CT_ASSIGNED: get_S_DS_Ids, - CT_INUSE: get_S_DS_Ids - }, - ServicePool: { - CT_ASSIGNED: get_Id, - CT_INUSE: get_Id, - CT_CACHED: get_Id - }, - Authenticator: { - CT_AUTH_USERS: get_Id, - CT_AUTH_SERVICES: get_Id, - CT_AUTH_USERS_WITH_SERVICES: get_Id, + __caRead.update( + { + Provider: { + CT_LOAD: get_Id, + CT_STORAGE: get_P_S_Ids, + CT_ASSIGNED: get_P_S_DS_Ids, + CT_INUSE: get_P_S_DS_Ids, + }, + Service: { + CT_STORAGE: get_Id, + CT_ASSIGNED: get_S_DS_Ids, + CT_INUSE: get_S_DS_Ids, + }, + ServicePool: {CT_ASSIGNED: get_Id, CT_INUSE: get_Id, CT_CACHED: get_Id}, + Authenticator: { + CT_AUTH_USERS: get_Id, + CT_AUTH_SERVICES: get_Id, + CT_AUTH_USERS_WITH_SERVICES: get_Id, + }, } - }) + ) - def _getIds(obj): + def _getIds(obj) -> typing.Tuple: to = type(obj) if to is ServicePool or to is Authenticator: return to.id if to is Service: - return (i.id for i in obj.userServices.all()) + return tuple(i.id for i in obj.userServices.all()) - res = () + res: typing.Tuple = () if to is Provider: for i in obj.services.all(): res += _getIds(i) @@ -200,23 +233,28 @@ def _initializeData() -> None: # Dict to convert objects to owner types # Dict for translations - __transDict.update({ - ServicePool: OT_DEPLOYED, - Service: OT_SERVICE, - Provider: OT_PROVIDER, - Authenticator: OT_AUTHENTICATOR, - }) + __transDict.update( + { + ServicePool: OT_DEPLOYED, + Service: OT_SERVICE, + Provider: OT_PROVIDER, + Authenticator: OT_AUTHENTICATOR, + } + ) # Titles of types - __typeTitles.update({ - CT_ASSIGNED: _('Assigned'), - CT_INUSE: _('In use'), - CT_LOAD: _('Load'), - CT_STORAGE: _('Storage'), - CT_AUTH_USERS: _('Users'), - CT_AUTH_USERS_WITH_SERVICES: _('Users with services'), - CT_AUTH_SERVICES: _('User Services'), - CT_CACHED: _('Cached'), - }) + __typeTitles.update( + { + CT_ASSIGNED: _('Assigned'), + CT_INUSE: _('In use'), + CT_LOAD: _('Load'), + CT_STORAGE: _('Storage'), + CT_AUTH_USERS: _('Users'), + CT_AUTH_USERS_WITH_SERVICES: _('Users with services'), + CT_AUTH_SERVICES: _('User Services'), + CT_CACHED: _('Cached'), + } + ) + _initializeData() diff --git a/server/src/uds/core/util/stats/events.py b/server/src/uds/core/util/stats/events.py index 153f65c97..e15edf82f 100644 --- a/server/src/uds/core/util/stats/events.py +++ b/server/src/uds/core/util/stats/events.py @@ -34,7 +34,7 @@ import datetime import logging import typing -from uds.core.managers import statsManager +from uds.core.managers.stats import StatsManager from uds.models import Provider, Service, ServicePool, Authenticator logger = logging.getLogger(__name__) @@ -128,7 +128,9 @@ def addEvent(obj: EventClass, eventType: int, **kwargs) -> bool: note: Runtime checks are done so if we try to insert an unssuported stat, this won't be inserted and it will be logged """ - return statsManager().addEvent(__transDict[type(obj)], obj.id, eventType, **kwargs) + return StatsManager.manager().addEvent( + __transDict[type(obj)], obj.id, eventType, **kwargs + ) def getEvents( @@ -160,7 +162,7 @@ def getEvents( else: owner_id = obj.pk - for i in statsManager().getEvents( + for i in StatsManager.manager().getEvents( __transDict[type_], eventType, owner_id=owner_id, since=since, to=to ): val = ( diff --git a/server/src/uds/core/workers/stats_collector.py b/server/src/uds/core/workers/stats_collector.py index 6d99562a8..929ed353e 100644 --- a/server/src/uds/core/workers/stats_collector.py +++ b/server/src/uds/core/workers/stats_collector.py @@ -35,7 +35,7 @@ import typing from uds.models import ServicePool, Authenticator, getSqlDatetime from uds.core.util.state import State from uds.core.util.stats import counters -from uds.core.managers import statsManager +from uds.core.managers.stats import StatsManager from uds.core.jobs import Job @@ -142,12 +142,12 @@ class StatsCleaner(Job): def run(self): logger.debug('Starting statistics cleanup') try: - statsManager().cleanupCounters() + StatsManager.manager().cleanupCounters() except Exception: logger.exception('Cleaning up counters') try: - statsManager().cleanupEvents() + StatsManager.manager().cleanupEvents() except Exception: logger.exception('Cleaning up events') diff --git a/server/src/uds/reports/stats/pool_users_summary.py b/server/src/uds/reports/stats/pool_users_summary.py index 8362d4649..45f22562b 100644 --- a/server/src/uds/reports/stats/pool_users_summary.py +++ b/server/src/uds/reports/stats/pool_users_summary.py @@ -39,6 +39,7 @@ from django.utils.translation import ugettext, ugettext_lazy as _ from uds.core.ui import gui from uds.core.util.stats import events +from uds.core.managers.stats import StatsManager from uds.models import ServicePool from .base import StatsReport @@ -88,7 +89,7 @@ class UsageSummaryByUsersPool(StatsReport): end = self.endDate.stamp() logger.debug(self.pool.value) - items = events.statsManager().getEvents(events.OT_DEPLOYED, (events.ET_LOGIN, events.ET_LOGOUT), owner_id=pool.id, since=start, to=end).order_by('stamp') + items = StatsManager.manager().getEvents(events.OT_DEPLOYED, (events.ET_LOGIN, events.ET_LOGOUT), owner_id=pool.id, since=start, to=end).order_by('stamp') logins: typing.Dict[str, int] = {} users: typing.Dict[str, typing.Dict] = {} diff --git a/server/src/uds/reports/stats/pools_performance.py b/server/src/uds/reports/stats/pools_performance.py index e702308e1..395038f44 100644 --- a/server/src/uds/reports/stats/pools_performance.py +++ b/server/src/uds/reports/stats/pools_performance.py @@ -43,6 +43,7 @@ import django.template.defaultfilters as filters from uds.core.ui import gui from uds.core.util.stats import events from uds.core.util import tools +from uds.core.managers.stats import StatsManager from uds.core.reports import graphs from uds.models import ServicePool @@ -147,7 +148,7 @@ class PoolPerformanceReport(StatsReport): # Store dataUsers for all pools poolsData = [] - fld = events.statsManager().getEventFldFor('username') + fld = StatsManager.manager().getEventFldFor('username') reportData = [] for p in pools: @@ -156,7 +157,7 @@ class PoolPerformanceReport(StatsReport): for interval in samplingIntervals: key = (interval[0] + interval[1]) / 2 q = ( - events.statsManager() + StatsManager.manager() .getEvents( events.OT_DEPLOYED, events.ET_ACCESS, diff --git a/server/src/uds/reports/stats/usage_by_pool.py b/server/src/uds/reports/stats/usage_by_pool.py index 3f4cb1421..e17708cdb 100644 --- a/server/src/uds/reports/stats/usage_by_pool.py +++ b/server/src/uds/reports/stats/usage_by_pool.py @@ -39,6 +39,7 @@ from django.utils.translation import ugettext, ugettext_lazy as _ from uds.core.ui import gui from uds.core.util.stats import events +from uds.core.managers.stats import StatsManager from uds.models import ServicePool from .base import StatsReport @@ -97,7 +98,7 @@ class UsageByPool(StatsReport): data = [] for pool in pools: items = ( - events.statsManager() + StatsManager.manager() .getEvents( events.OT_DEPLOYED, (events.ET_LOGIN, events.ET_LOGOUT), diff --git a/server/src/uds/reports/stats/user_access.py b/server/src/uds/reports/stats/user_access.py index 6017b7f8e..3e3712020 100644 --- a/server/src/uds/reports/stats/user_access.py +++ b/server/src/uds/reports/stats/user_access.py @@ -41,6 +41,7 @@ import django.template.defaultfilters as filters from uds.core.ui import gui from uds.core.util.stats import events from uds.core.util import tools +from uds.core.managers.stats import StatsManager from uds.core.reports import graphs from .base import StatsReport @@ -126,7 +127,7 @@ class StatsReportLogin(StatsReport): for interval in samplingIntervals: key = (interval[0] + interval[1]) / 2 val = ( - events.statsManager() + StatsManager.manager() .getEvents( events.OT_AUTHENTICATOR, events.ET_LOGIN, @@ -154,7 +155,7 @@ class StatsReportLogin(StatsReport): dataWeek = [0] * 7 dataHour = [0] * 24 dataWeekHour = [[0] * 24 for _ in range(7)] - for val in events.statsManager().getEvents( + for val in StatsManager.manager().getEvents( events.OT_AUTHENTICATOR, events.ET_LOGIN, since=start, to=end ): s = datetime.datetime.fromtimestamp(val.stamp)