diff --git a/server/src/uds/core/util/auto_attributes.py b/server/src/uds/core/util/auto_attributes.py index 8f7a55608..a094ff461 100644 --- a/server/src/uds/core/util/auto_attributes.py +++ b/server/src/uds/core/util/auto_attributes.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -83,13 +83,13 @@ class AutoAttributes(Serializable): def __getattribute__(self, name): if name.startswith('_') and name[1:] in self.attrs: return self.attrs[name[1:]].getValue() - return object.__getattribute__(self, name) + return super().__getattribute__(name) def __setattr__(self, name, value): if name.startswith('_') and name[1:] in self.attrs: self.attrs[name[1:]].setValue(value) else: - object.__setattr__(self, name, value) + super().__setattr__(name, value) def declare(self, **kwargs): d = {} @@ -100,7 +100,15 @@ class AutoAttributes(Serializable): def marshal(self) -> bytes: for k, v in self.attrs.items(): logger.debug('Marshall Autoattributes: %s=%s', k, v.getValue()) - return codecs.encode(b'\2'.join([b'%s\1%s' % (k.encode('utf8'), pickle.dumps(v, protocol=0)) for k, v in self.attrs.items()]), 'bz2') + return codecs.encode( + b'\2'.join( + [ + b'%s\1%s' % (k.encode('utf8'), pickle.dumps(v, protocol=0)) + for k, v in self.attrs.items() + ] + ), + 'bz2', + ) def unmarshal(self, data: bytes) -> None: if not data: # Can be empty @@ -116,7 +124,7 @@ class AutoAttributes(Serializable): # logger.debug('k: %s --- v: %s', k, v) try: self.attrs[k.decode()] = pickle.loads(v) - except Exception: # Old encoding on python2, set encoding for loading + except Exception: # Old encoding on python2, set encoding for loading self.attrs[k.decode()] = pickle.loads(v, encoding='utf8') for k2, v2 in self.attrs.items(): diff --git a/server/src/uds/core/util/config.py b/server/src/uds/core/util/config.py index d0e856058..6038cbaec 100644 --- a/server/src/uds/core/util/config.py +++ b/server/src/uds/core/util/config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -11,7 +11,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -63,8 +63,15 @@ class Config: HIDDEN_FIELD: int = 6 # Not visible on "admin" config edition class Value: - - def __init__(self, section: 'Config.Section', key: str, default: str = '', crypt: bool = False, longText: bool = False, **kwargs): + def __init__( + self, + section: 'Config.Section', + key: str, + default: str = '', + crypt: bool = False, + longText: bool = False, + **kwargs + ): logger.debug('Var: %s %s KWARGS: %s', section, key, kwargs) self._type: int = kwargs.get('type', -1) @@ -92,9 +99,13 @@ class Config: try: if force or self._data is None: # logger.debug('Accessing db config {0}.{1}'.format(self._section.name(), self._key)) - readed = DBConfig.objects.get(section=self._section.name(), key=self._key) # @UndefinedVariable + readed = DBConfig.objects.get( + section=self._section.name(), key=self._key + ) # @UndefinedVariable self._data = readed.value - self._crypt = [self._crypt, True][readed.crypt] # True has "higher" precedende than False + self._crypt = [self._crypt, True][ + readed.crypt + ] # True has "higher" precedende than False self._longText = readed.long if self._type != -1 and self._type != readed.field_type: readed.field_type = self._type @@ -119,11 +130,19 @@ class Config: try: return int(self.get(force)) except Exception: - logger.error('Value for %s.%s is invalid (integer expected)', self._section, self._key) + logger.error( + 'Value for %s.%s is invalid (integer expected)', + self._section, + self._key, + ) try: return int(self._default) except Exception: - logger.error('Default value for %s.%s is also invalid (integer expected)', self._section, self._key) + logger.error( + 'Default value for %s.%s is also invalid (integer expected)', + self._section, + self._key, + ) return -1 def getBool(self, force: bool = False) -> bool: @@ -159,17 +178,32 @@ class Config: # Editable here means that this configuration value can be edited by admin directly (generally, that this is a "clean text" value) - logger.debug('Saving config %s.%s as %s', self._section.name(), self._key, value) + logger.debug( + 'Saving config %s.%s as %s', self._section.name(), self._key, value + ) try: - obj, _ = DBConfig.objects.get_or_create(section=self._section.name(), key=self._key) # @UndefinedVariable - obj.value, obj.crypt, obj.long, obj.field_type = value, self._crypt, self._longText, self._type + obj, _ = DBConfig.objects.get_or_create( + section=self._section.name(), key=self._key + ) # @UndefinedVariable + obj.value, obj.crypt, obj.long, obj.field_type = ( + value, + self._crypt, + self._longText, + self._type, + ) obj.save() except Exception: - if 'migrate' in sys.argv: # During migration, set could be saved as part of initialization... + if ( + 'migrate' in sys.argv + ): # During migration, set could be saved as part of initialization... return logger.exception('Exception') # Probably a migration issue, just ignore it - logger.info("Could not save configuration key %s.%s", self._section.name(), self._key) + logger.info( + "Could not save configuration key %s.%s", + self._section.name(), + self._key, + ) class Section: def __init__(self, sectionName: str): @@ -192,7 +226,14 @@ class Config: return Config.Section(sectionName) @staticmethod - def value(section: Section, key: str, default: str, crypt: bool = False, longText: bool = False, **kwargs) -> 'Config.Value': + def value( + section: Section, + key: str, + default: str, + crypt: bool = False, + longText: bool = False, + **kwargs + ) -> 'Config.Value': return Config.Value(section, key, default, crypt, longText, **kwargs) @staticmethod @@ -216,9 +257,11 @@ class Config: def update(section, key, value, checkType=False) -> bool: # If cfg value does not exists, simply ignore request try: - cfg = DBConfig.objects.filter(section=section, key=key)[0] # @UndefinedVariable + cfg = DBConfig.objects.filter(section=section, key=key)[ + 0 + ] # @UndefinedVariable if checkType and cfg.field_type in (Config.READ_FIELD, Config.HIDDEN_FIELD): - return False # Skip non writable elements + return False # Skip non writable elements if cfg.crypt: value = cryptoManager().encrypt(value) @@ -234,121 +277,224 @@ class GlobalConfig: """ Simple helper to keep track of global configuration """ - SESSION_EXPIRE_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('sessionExpireTime', '24', type=Config.NUMERIC_FIELD) # Max session duration (in use) after a new publishment has been made + + SESSION_EXPIRE_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'sessionExpireTime', '24', type=Config.NUMERIC_FIELD + ) # Max session duration (in use) after a new publishment has been made # Delay between cache checks. reducing this number will increase cache generation speed but also will load service providers - CACHE_CHECK_DELAY: Config.Value = Config.section(GLOBAL_SECTION).value('cacheCheckDelay', '19', type=Config.NUMERIC_FIELD) + CACHE_CHECK_DELAY: Config.Value = Config.section(GLOBAL_SECTION).value( + 'cacheCheckDelay', '19', type=Config.NUMERIC_FIELD + ) # Delayed task number of threads PER SERVER, with higher number of threads, deplayed task will complete sooner, but it will give more load to overall system - DELAYED_TASKS_THREADS: Config.Value = Config.section(GLOBAL_SECTION).value('delayedTasksThreads', '4', type=Config.NUMERIC_FIELD) + DELAYED_TASKS_THREADS: Config.Value = Config.section(GLOBAL_SECTION).value( + 'delayedTasksThreads', '4', type=Config.NUMERIC_FIELD + ) # Number of scheduler threads running PER SERVER, with higher number of threads, deplayed task will complete sooner, but it will give more load to overall system - SCHEDULER_THREADS: Config.Value = Config.section(GLOBAL_SECTION).value('schedulerThreads', '3', type=Config.NUMERIC_FIELD) + SCHEDULER_THREADS: Config.Value = Config.section(GLOBAL_SECTION).value( + 'schedulerThreads', '3', type=Config.NUMERIC_FIELD + ) # Waiting time before removing "errored" and "removed" publications, cache, and user assigned machines. Time is in seconds - CLEANUP_CHECK: Config.Value = Config.section(GLOBAL_SECTION).value('cleanupCheck', '3607', type=Config.NUMERIC_FIELD) + CLEANUP_CHECK: Config.Value = Config.section(GLOBAL_SECTION).value( + 'cleanupCheck', '3607', type=Config.NUMERIC_FIELD + ) # Time to maintaing "info state" items before removing it, in seconds - KEEP_INFO_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('keepInfoTime', '14401', type=Config.NUMERIC_FIELD) # Defaults to 2 days 172800?? better 4 hours xd + KEEP_INFO_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'keepInfoTime', '14401', type=Config.NUMERIC_FIELD + ) # Defaults to 2 days 172800?? better 4 hours xd # Max number of services to be "preparing" at same time - MAX_PREPARING_SERVICES: Config.Value = Config.section(GLOBAL_SECTION).value('maxPreparingServices', '15', type=Config.NUMERIC_FIELD) # Defaults to 15 services at once (per service provider) + MAX_PREPARING_SERVICES: Config.Value = Config.section(GLOBAL_SECTION).value( + 'maxPreparingServices', '15', type=Config.NUMERIC_FIELD + ) # Defaults to 15 services at once (per service provider) # Max number of service to be at "removal" state at same time - MAX_REMOVING_SERVICES: Config.Value = Config.section(GLOBAL_SECTION).value('maxRemovingServices', '15', type=Config.NUMERIC_FIELD) # Defaults to 15 services at once (per service provider) + MAX_REMOVING_SERVICES: Config.Value = Config.section(GLOBAL_SECTION).value( + 'maxRemovingServices', '15', type=Config.NUMERIC_FIELD + ) # Defaults to 15 services at once (per service provider) # If we ignore limits (max....) - IGNORE_LIMITS: Config.Value = Config.section(GLOBAL_SECTION).value('ignoreLimits', '0', type=Config.BOOLEAN_FIELD) + IGNORE_LIMITS: Config.Value = Config.section(GLOBAL_SECTION).value( + 'ignoreLimits', '0', type=Config.BOOLEAN_FIELD + ) # Number of services to initiate removal per run of CacheCleaner - USER_SERVICE_CLEAN_NUMBER: Config.Value = Config.section(GLOBAL_SECTION).value('userServiceCleanNumber', '8', type=Config.NUMERIC_FIELD) # Defaults to 3 per wun + USER_SERVICE_CLEAN_NUMBER: Config.Value = Config.section(GLOBAL_SECTION).value( + 'userServiceCleanNumber', '8', type=Config.NUMERIC_FIELD + ) # Defaults to 3 per wun # Removal Check time for cache, publications and deployed services - REMOVAL_CHECK: Config.Value = Config.section(GLOBAL_SECTION).value('removalCheck', '31', type=Config.NUMERIC_FIELD) # Defaults to 30 seconds + REMOVAL_CHECK: Config.Value = Config.section(GLOBAL_SECTION).value( + 'removalCheck', '31', type=Config.NUMERIC_FIELD + ) # Defaults to 30 seconds # Login URL: deprecated & not used anymore # LOGIN_URL: Config.Value = Config.section(GLOBAL_SECTION).value('loginUrl', '/uds/page/login', type=Config.TEXT_FIELD) # Defaults to /login # Session duration # USER_SESSION_LENGTH: Config.Value = Config.section(SECURITY_SECTION).value('userSessionLength', '14400', type=Config.NUMERIC_FIELD) # Defaults to 4 hours # Superuser (do not need to be at database!!!) - SUPER_USER_LOGIN: Config.Value = Config.section(SECURITY_SECTION).value('superUser', 'root', type=Config.TEXT_FIELD) + SUPER_USER_LOGIN: Config.Value = Config.section(SECURITY_SECTION).value( + 'superUser', 'root', type=Config.TEXT_FIELD + ) # Superuser password (do not need to be at database!!!) - SUPER_USER_PASS: Config.Value = Config.section(SECURITY_SECTION).valueCrypt('rootPass', 'udsmam0', type=Config.TEXT_FIELD) + SUPER_USER_PASS: Config.Value = Config.section(SECURITY_SECTION).valueCrypt( + 'rootPass', 'udsmam0', type=Config.TEXT_FIELD + ) # Idle time before closing session on admin - SUPER_USER_ALLOW_WEBACCESS: Config.Value = Config.section(SECURITY_SECTION).value('allowRootWebAccess', '1', type=Config.BOOLEAN_FIELD) + SUPER_USER_ALLOW_WEBACCESS: Config.Value = Config.section(SECURITY_SECTION).value( + 'allowRootWebAccess', '1', type=Config.BOOLEAN_FIELD + ) # Time an admi session can be idle before being "logged out" # ADMIN_IDLE_TIME: Config.Value = Config.section(SECURITY_SECTION).value('adminIdleTime', '14400', type=Config.NUMERIC_FIELD) # Defaults to 4 hous # Time betwen checks of unused services by os managers # Unused services will be invoked for every machine assigned but not in use AND that has been assigned at least this time # (only if os manager asks for this characteristic) - CHECK_UNUSED_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('checkUnusedTime', '631', type=Config.NUMERIC_FIELD) # Defaults to 10 minutes - CHECK_UNUSED_DELAY: Config.Value = Config.section(GLOBAL_SECTION).value('checkUnusedDelay', '300', type=Config.NUMERIC_FIELD) # Defaults to 10 minutes + CHECK_UNUSED_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'checkUnusedTime', '631', type=Config.NUMERIC_FIELD + ) # Defaults to 10 minutes + CHECK_UNUSED_DELAY: Config.Value = Config.section(GLOBAL_SECTION).value( + 'checkUnusedDelay', '300', type=Config.NUMERIC_FIELD + ) # Defaults to 10 minutes # Default CSS Used: REMOVED! (keep the for for naw, for reference, but will be cleaned on future...) # CSS: Config.Value = Config.section(GLOBAL_SECTION).value('css', settings.STATIC_URL + 'css/uds.css', type=Config.TEXT_FIELD) # Max logins before blocking an account - MAX_LOGIN_TRIES: Config.Value = Config.section(SECURITY_SECTION).value('maxLoginTries', '5', type=Config.NUMERIC_FIELD) + MAX_LOGIN_TRIES: Config.Value = Config.section(SECURITY_SECTION).value( + 'maxLoginTries', '5', type=Config.NUMERIC_FIELD + ) # Block time in second for an user that makes too many mistakes, 5 minutes default - LOGIN_BLOCK: Config.Value = Config.section(SECURITY_SECTION).value('loginBlockTime', '300', type=Config.NUMERIC_FIELD) - LOGIN_BLOCK_IP: Config.Value = Config.section(SECURITY_SECTION).value('Block ip on login failure', '0', type=Config.BOOLEAN_FIELD) + LOGIN_BLOCK: Config.Value = Config.section(SECURITY_SECTION).value( + 'loginBlockTime', '300', type=Config.NUMERIC_FIELD + ) + LOGIN_BLOCK_IP: Config.Value = Config.section(SECURITY_SECTION).value( + 'Block ip on login failure', '0', type=Config.BOOLEAN_FIELD + ) # Do autorun of service if just one service. # 0 = No autorun, 1 = Autorun at login # In a future, maybe necessary another value "2" that means that autorun always - AUTORUN_SERVICE: Config.Value = Config.section(GLOBAL_SECTION).value('autorunService', '0', type=Config.BOOLEAN_FIELD) + AUTORUN_SERVICE: Config.Value = Config.section(GLOBAL_SECTION).value( + 'autorunService', '0', type=Config.BOOLEAN_FIELD + ) # Redirect HTTP to HTTPS - REDIRECT_TO_HTTPS: Config.Value = Config.section(GLOBAL_SECTION).value('redirectToHttps', '0', type=Config.BOOLEAN_FIELD) + REDIRECT_TO_HTTPS: Config.Value = Config.section(GLOBAL_SECTION).value( + 'redirectToHttps', '0', type=Config.BOOLEAN_FIELD + ) # Max time needed to get a service "fully functional" before it's considered "failed" and removed # The time is in seconds - MAX_INITIALIZING_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('maxInitTime', '3601', type=Config.NUMERIC_FIELD) - MAX_REMOVAL_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('maxRemovalTime', '86400', type=Config.NUMERIC_FIELD) + MAX_INITIALIZING_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'maxInitTime', '3601', type=Config.NUMERIC_FIELD + ) + MAX_REMOVAL_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'maxRemovalTime', '86400', type=Config.NUMERIC_FIELD + ) # Maximum logs per user service - MAX_LOGS_PER_ELEMENT: Config.Value = Config.section(GLOBAL_SECTION).value('maxLogPerElement', '100', type=Config.NUMERIC_FIELD) + MAX_LOGS_PER_ELEMENT: Config.Value = Config.section(GLOBAL_SECTION).value( + 'maxLogPerElement', '100', type=Config.NUMERIC_FIELD + ) # Time to restrain a deployed service in case it gives some errors at some point - RESTRAINT_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('restrainTime', '600', type=Config.NUMERIC_FIELD) + RESTRAINT_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'restrainTime', '600', type=Config.NUMERIC_FIELD + ) # Number of errors that must occurr in RESTRAIN_TIME to restrain deployed service - RESTRAINT_COUNT: Config.Value = Config.section(GLOBAL_SECTION).value('restrainCount', '3', type=Config.NUMERIC_FIELD) + RESTRAINT_COUNT: Config.Value = Config.section(GLOBAL_SECTION).value( + 'restrainCount', '3', type=Config.NUMERIC_FIELD + ) # Statistics duration, in days - STATS_DURATION: Config.Value = Config.section(GLOBAL_SECTION).value('statsDuration', '365', type=Config.NUMERIC_FIELD) + STATS_DURATION: Config.Value = Config.section(GLOBAL_SECTION).value( + 'statsDuration', '365', type=Config.NUMERIC_FIELD + ) # If disallow login using /login url, and must go to an authenticator - DISALLOW_GLOBAL_LOGIN: Config.Value = Config.section(GLOBAL_SECTION).value('disallowGlobalLogin', '0', type=Config.BOOLEAN_FIELD) + DISALLOW_GLOBAL_LOGIN: Config.Value = Config.section(GLOBAL_SECTION).value( + 'disallowGlobalLogin', '0', type=Config.BOOLEAN_FIELD + ) # Allos preferences access to users - NOTIFY_REMOVAL_BY_PUB: Config.Value = Config.section(GLOBAL_SECTION).value('Notify on new publication', '0', type=Config.BOOLEAN_FIELD) + NOTIFY_REMOVAL_BY_PUB: Config.Value = Config.section(GLOBAL_SECTION).value( + 'Notify on new publication', '0', type=Config.BOOLEAN_FIELD + ) # Allowed "trusted sources" for request - TRUSTED_SOURCES: Config.Value = Config.section(SECURITY_SECTION).value('Trusted Hosts', '*', type=Config.TEXT_FIELD) + TRUSTED_SOURCES: Config.Value = Config.section(SECURITY_SECTION).value( + 'Trusted Hosts', '*', type=Config.TEXT_FIELD + ) # Allow clients to notify their own ip (if set), or use always the request extracted IP - HONOR_CLIENT_IP_NOTIFY: Config.Value = Config.section(SECURITY_SECTION).value('honorClientNotifyIP', '0', type=Config.BOOLEAN_FIELD) + HONOR_CLIENT_IP_NOTIFY: Config.Value = Config.section(SECURITY_SECTION).value( + 'honorClientNotifyIP', '0', type=Config.BOOLEAN_FIELD + ) # If there is a proxy in front of us - BEHIND_PROXY: Config.Value = Config.section(SECURITY_SECTION).value('Behind a proxy', '0', type=Config.BOOLEAN_FIELD) + BEHIND_PROXY: Config.Value = Config.section(SECURITY_SECTION).value( + 'Behind a proxy', '0', type=Config.BOOLEAN_FIELD + ) # If we use new logout mechanics - EXCLUSIVE_LOGOUT: Config.Value = Config.section(SECURITY_SECTION).value('Exclusive Logout', '0', type=Config.BOOLEAN_FIELD) + EXCLUSIVE_LOGOUT: Config.Value = Config.section(SECURITY_SECTION).value( + 'Exclusive Logout', '0', type=Config.BOOLEAN_FIELD + ) # Enable/Disable Actor attack detection ip blocking - BLOCK_ACTOR_FAILURES: Config.Value = Config.section(SECURITY_SECTION).value('Block actor failures', '1', type=Config.BOOLEAN_FIELD) + BLOCK_ACTOR_FAILURES: Config.Value = Config.section(SECURITY_SECTION).value( + 'Block actor failures', '1', type=Config.BOOLEAN_FIELD + ) # Max session length configuration values - SESSION_DURATION_ADMIN: Config.Value = Config.section(SECURITY_SECTION).value('Session timeout for Admin', '14400', type=Config.NUMERIC_FIELD) - SESSION_DURATION_USER: Config.Value = Config.section(SECURITY_SECTION).value('Session timeout for User', '14400', type=Config.NUMERIC_FIELD) + SESSION_DURATION_ADMIN: Config.Value = Config.section(SECURITY_SECTION).value( + 'Session timeout for Admin', '14400', type=Config.NUMERIC_FIELD + ) + SESSION_DURATION_USER: Config.Value = Config.section(SECURITY_SECTION).value( + 'Session timeout for User', '14400', type=Config.NUMERIC_FIELD + ) - RELOAD_TIME: Config.Value = Config.section(GLOBAL_SECTION).value('Page reload Time', '300', type=Config.NUMERIC_FIELD) + RELOAD_TIME: Config.Value = Config.section(GLOBAL_SECTION).value( + 'Page reload Time', '300', type=Config.NUMERIC_FIELD + ) - LIMITED_BY_CALENDAR_TEXT: Config.Value = Config.section(GLOBAL_SECTION).value('Calendar access denied text', '', type=Config.TEXT_FIELD) # Defaults to Nothing + LIMITED_BY_CALENDAR_TEXT: Config.Value = Config.section(GLOBAL_SECTION).value( + 'Calendar access denied text', '', type=Config.TEXT_FIELD + ) # Defaults to Nothing # Custom message for error when limiting by calendar # If convert username to lowercase - LOWERCASE_USERNAME: Config.Value = Config.section(SECURITY_SECTION).value('Convert username to lowercase', '1', type=Config.BOOLEAN_FIELD) + LOWERCASE_USERNAME: Config.Value = Config.section(SECURITY_SECTION).value( + 'Convert username to lowercase', '1', type=Config.BOOLEAN_FIELD + ) # Global UDS ID (common for all servers on the same cluster) - UDS_ID: Config.Value = Config.section(GLOBAL_SECTION).value('UDS ID', cryptoManager().uuid(), type=Config.READ_FIELD) + UDS_ID: Config.Value = Config.section(GLOBAL_SECTION).value( + 'UDS ID', cryptoManager().uuid(), type=Config.READ_FIELD + ) # Site display name & copyright info - SITE_NAME: Config.Value = Config.section(CUSTOM_SECTION).value('Site name', 'UDS Enterprise', type=Config.TEXT_FIELD) - SITE_COPYRIGHT: Config.Value = Config.section(CUSTOM_SECTION).value('Site copyright info', '© Virtual Cable S.L.U.', type=Config.TEXT_FIELD) - SITE_COPYRIGHT_LINK: Config.Value = Config.section(CUSTOM_SECTION).value('Site copyright link', 'https://www.udsenterprise.com', type=Config.TEXT_FIELD) - SITE_LOGO_NAME: Config.Value = Config.section(CUSTOM_SECTION).value('Logo name', 'UDS', type=Config.TEXT_FIELD) - SITE_CSS: Config.Value = Config.section(CUSTOM_SECTION).value('CSS', '', type=Config.LONGTEXT_FIELD) - SITE_INFO: Config.Value = Config.section(CUSTOM_SECTION).value('Site information', '', type=Config.LONGTEXT_FIELD) - SITE_FILTER_ONTOP: Config.Value = Config.section(CUSTOM_SECTION).value('Show Filter on Top', '0', type=Config.BOOLEAN_FIELD) - SITE_FILTER_MIN: Config.Value = Config.section(CUSTOM_SECTION).value('Min. Services to show filter', '8', type=Config.NUMERIC_FIELD) + SITE_NAME: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Site name', 'UDS Enterprise', type=Config.TEXT_FIELD + ) + SITE_COPYRIGHT: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Site copyright info', '© Virtual Cable S.L.U.', type=Config.TEXT_FIELD + ) + SITE_COPYRIGHT_LINK: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Site copyright link', 'https://www.udsenterprise.com', type=Config.TEXT_FIELD + ) + SITE_LOGO_NAME: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Logo name', 'UDS', type=Config.TEXT_FIELD + ) + SITE_CSS: Config.Value = Config.section(CUSTOM_SECTION).value( + 'CSS', '', type=Config.LONGTEXT_FIELD + ) + SITE_INFO: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Site information', '', type=Config.LONGTEXT_FIELD + ) + SITE_FILTER_ONTOP: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Show Filter on Top', '0', type=Config.BOOLEAN_FIELD + ) + SITE_FILTER_MIN: Config.Value = Config.section(CUSTOM_SECTION).value( + 'Min. Services to show filter', '8', type=Config.NUMERIC_FIELD + ) - EXPERIMENTAL_FEATURES: Config.Value = Config.section(GLOBAL_SECTION).value('Experimental Features', '0', type=Config.BOOLEAN_FIELD) + EXPERIMENTAL_FEATURES: Config.Value = Config.section(GLOBAL_SECTION).value( + 'Experimental Features', '0', type=Config.BOOLEAN_FIELD + ) # Admin config variables - ADMIN_PAGESIZE: Config.Value = Config.section(ADMIN_SECTION).value('List page size', '10', type=Config.NUMERIC_FIELD) - ADMIN_TRUSTED_SOURCES: Config.Value = Config.section(ADMIN_SECTION).value('Trusted Hosts for Admin', '*', type=Config.TEXT_FIELD) + ADMIN_PAGESIZE: Config.Value = Config.section(ADMIN_SECTION).value( + 'List page size', '10', type=Config.NUMERIC_FIELD + ) + ADMIN_TRUSTED_SOURCES: Config.Value = Config.section(ADMIN_SECTION).value( + 'Trusted Hosts for Admin', '*', type=Config.TEXT_FIELD + ) _initDone = False @@ -381,4 +527,6 @@ class GlobalConfig: # GlobalConfig.UDS_THEME.setParams(['html5', 'semantic']) except Exception: - logger.debug('Config table do not exists!!!, maybe we are installing? :-)') + logger.debug( + 'Config table do not exists!!!, maybe we are installing? :-)' + ) diff --git a/server/src/uds/core/util/connection.py b/server/src/uds/core/util/connection.py index a3f2ec437..668172eea 100644 --- a/server/src/uds/core/util/connection.py +++ b/server/src/uds/core/util/connection.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -39,10 +39,14 @@ logger = logging.getLogger(__name__) def testServer(host: str, port: typing.Union[int, str], timeOut: float = 4) -> bool: try: - logger.debug('Checking connection to %s:%s with %s seconds timeout', host, port, timeOut) + logger.debug( + 'Checking connection to %s:%s with %s seconds timeout', host, port, timeOut + ) sock = socket.create_connection((host, int(port)), timeOut) sock.close() except Exception as e: - logger.debug('Exception checking %s:%s with %s timeout: %s', host, port, timeOut, e) + logger.debug( + 'Exception checking %s:%s with %s timeout: %s', host, port, timeOut, e + ) return False return True diff --git a/server/src/uds/core/util/custom.py b/server/src/uds/core/util/custom.py deleted file mode 100644 index 2eb109f65..000000000 --- a/server/src/uds/core/util/custom.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -# -# Copyright (c) 2012-2019 Virtual Cable S.L. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""" -.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com -""" - -# Default values -STYLESHEET_DEFAULT = ''' -.mat-toolbar.mat-primary { - background-color: #455A64; -} -''' - -ICON_DEFAULT = ''' - -''' \ No newline at end of file diff --git a/server/src/uds/core/util/decorators.py b/server/src/uds/core/util/decorators.py index 6f547eaeb..720b07db9 100644 --- a/server/src/uds/core/util/decorators.py +++ b/server/src/uds/core/util/decorators.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -46,9 +46,11 @@ RT = typing.TypeVar('RT') # Decorator that protects pages that needs at least a browser version # Default is to deny IE < 9 def denyBrowsers( - browsers: typing.Optional[typing.List[str]] = None, - errorResponse: typing.Callable = lambda request: errors.errorView(request, errors.BROWSER_NOT_SUPPORTED) - ) -> typing.Callable[[typing.Callable[..., RT]], typing.Callable[..., RT]]: + browsers: typing.Optional[typing.List[str]] = None, + errorResponse: typing.Callable = lambda request: errors.errorView( + request, errors.BROWSER_NOT_SUPPORTED + ), +) -> typing.Callable[[typing.Callable[..., RT]], typing.Callable[..., RT]]: """ Decorator to set protection to access page Look for samples at uds.core.web.views @@ -82,7 +84,12 @@ def deprecated(func: typing.Callable[..., RT]) -> typing.Callable[..., RT]: def new_func(*args, **kwargs) -> RT: try: caller = inspect.stack()[1] - logger.warning('Call to deprecated function %s from %s:%s.', func.__name__, caller[1], caller[2]) + logger.warning( + 'Call to deprecated function %s from %s:%s.', + func.__name__, + caller[1], + caller[2], + ) except Exception: logger.info('No stack info on deprecated function call %s', func.__name__) @@ -92,28 +99,31 @@ def deprecated(func: typing.Callable[..., RT]) -> typing.Callable[..., RT]: def ensureConected(func: typing.Callable[..., RT]) -> typing.Callable[..., RT]: - """This decorator calls "connect" method of the class of the wrapped object - """ + """This decorator calls "connect" method of the class of the wrapped object""" + @wraps(func) def new_func(*args, **kwargs) -> RT: args[0].connect() return func(*args, **kwargs) - + return new_func - # Decorator that allows us a "fast&clean" caching system on service providers # # Decorator for caching # Decorator that tries to get from cache before executing def allowCache( - cachePrefix: str, - cacheTimeout: int, - cachingArgs: typing.Optional[typing.Union[typing.List[int], typing.Tuple[int], int]] = None, - cachingKWArgs: typing.Optional[typing.Union[typing.List[str], typing.Tuple[str], str]] = None, - cachingKeyFnc: typing.Optional[typing.Callable[[typing.Any], str]] = None - ) -> typing.Callable[[typing.Callable[..., RT]], typing.Callable[..., RT]]: + cachePrefix: str, + cacheTimeout: int, + cachingArgs: typing.Optional[ + typing.Union[typing.List[int], typing.Tuple[int], int] + ] = None, + cachingKWArgs: typing.Optional[ + typing.Union[typing.List[str], typing.Tuple[str], str] + ] = None, + cachingKeyFnc: typing.Optional[typing.Callable[[typing.Any], str]] = None, +) -> typing.Callable[[typing.Callable[..., RT]], typing.Callable[..., RT]]: """Decorator that give us a "quick& clean" caching feature on service providers. Note: This decorator is intended ONLY for service providers @@ -130,11 +140,19 @@ def allowCache( def wrapper(*args, **kwargs) -> RT: argList: typing.List[str] = [] if cachingArgs: - ar = [cachingArgs] if not isinstance(cachingArgs, (list, tuple)) else cachingArgs + ar = ( + [cachingArgs] + if not isinstance(cachingArgs, (list, tuple)) + else cachingArgs + ) argList = [args[i] if i < len(args) else '' for i in ar] if cachingKWArgs: - kw = [cachingKWArgs] if not isinstance(cachingKWArgs, (list, tuple)) else cachingKWArgs + kw = ( + [cachingKWArgs] + if not isinstance(cachingKWArgs, (list, tuple)) + else cachingKWArgs + ) argList += [str(kwargs.get(i, '')) for i in kw] if argList: @@ -156,7 +174,13 @@ def allowCache( # Maybe returned data is not serializable. In that case, cache will fail but no harm is done with this args[0].cache.put(cacheKey, data, cacheTimeout) except Exception as e: - logger.debug('Data for %s is not serializable on call to %s, not cached. %s (%s)', cacheKey, fnc.__name__, data, e) + logger.debug( + 'Data for %s is not serializable on call to %s, not cached. %s (%s)', + cacheKey, + fnc.__name__, + data, + e, + ) return data return wrapper diff --git a/server/src/uds/core/util/file_storage.py b/server/src/uds/core/util/file_storage.py index 182c97dcb..f375739ec 100644 --- a/server/src/uds/core/util/file_storage.py +++ b/server/src/uds/core/util/file_storage.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2016-2019 Virtual Cable S.L. +# Copyright (c) 2016-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -88,9 +88,9 @@ class FileStorage(Storage): def _getKey(self, name): """ - We have only a few files on db, an we are running on a 64 bits system - memcached does not allow keys bigger than 250 chars, so we are going to use hash() to - get a key for this + We have only a few files on db, an we are running on a 64 bits system + memcached does not allow keys bigger than 250 chars, so we are going to use hash() to + get a key for this """ return 'fstor' + str(hash(self.get_valid_name(name))) @@ -112,14 +112,16 @@ class FileStorage(Storage): if self.cache is None: return - dbf = DictAsObj({ - 'name': f.name, - 'uuid': f.uuid, - 'size': f.size, - 'data': f.data, - 'created': f.created, - 'modified': f.modified - }) + dbf = DictAsObj( + { + 'name': f.name, + 'uuid': f.uuid, + 'size': f.size, + 'data': f.data, + 'created': f.created, + 'modified': f.modified, + } + ) self.cache.set(self._getKey(f.name), dbf, 3600) # Cache defaults to one hour @@ -131,7 +133,6 @@ class FileStorage(Storage): def _open(self, name, mode='rb'): f = io.BytesIO(self._dbFileForReadOnly(name).data) f.name = name - f.mode = mode return File(f) def _save(self, name, content): @@ -140,7 +141,9 @@ class FileStorage(Storage): f = self._dbFileForReadWrite(name) except DBFile.DoesNotExist: now = getSqlDatetime() - f = DBFile.objects.create(owner=self.owner, name=name, created=now, modified=now) + f = DBFile.objects.create( + owner=self.owner, name=name, created=now, modified=now + ) f.data = content.read() f.modified = getSqlDatetime() diff --git a/server/src/uds/core/util/html.py b/server/src/uds/core/util/html.py index 9410d239a..ee1499163 100644 --- a/server/src/uds/core/util/html.py +++ b/server/src/uds/core/util/html.py @@ -59,10 +59,14 @@ def udsLink(request: 'HttpRequest', ticket: str, scrambler: str) -> str: else: proto = 'uds' - return "{}://{}{}/{}".format(proto, request.build_absolute_uri('/').split('//')[1], ticket, scrambler) + return "{}://{}{}/{}".format( + proto, request.build_absolute_uri('/').split('//')[1], ticket, scrambler + ) -def udsAccessLink(request: 'HttpRequest', serviceId: str, transportId: typing.Optional[str]) -> str: +def udsAccessLink( + request: 'HttpRequest', serviceId: str, transportId: typing.Optional[str] +) -> str: ''' If transportId (uuid) is None, this will be a metaLink ''' @@ -73,7 +77,12 @@ def parseDate(dateToParse) -> datetime.date: if get_language() == 'fr': date_format = '%d/%m/%Y' else: - date_format = formats.get_format('SHORT_DATE_FORMAT').replace('Y', '%Y').replace('m', '%m').replace('d', '%d') # pylint: disable=maybe-no-member + date_format = ( + formats.get_format('SHORT_DATE_FORMAT') + .replace('Y', '%Y') + .replace('m', '%m') + .replace('d', '%d') + ) # pylint: disable=maybe-no-member return datetime.datetime.strptime(dateToParse, date_format).date() @@ -114,7 +123,7 @@ def checkBrowser(request: 'ExtendedHttpRequest', browser: str) -> bool: if browser.startswith(b): if request.os['Browser'] not in requires: return False - browser = browser[len(b):] # remove "browser name" from string + browser = browser[len(b) :] # remove "browser name" from string break browser += ' ' # So we ensure we have at least beowser[0] diff --git a/server/src/uds/core/util/log.py b/server/src/uds/core/util/log.py index 0ea6ce063..c778acb3f 100644 --- a/server/src/uds/core/util/log.py +++ b/server/src/uds/core/util/log.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -43,12 +43,30 @@ logger = logging.getLogger(__name__) useLogger = logging.getLogger('useLog') # Logging levels -OTHER, DEBUG, INFO, WARN, ERROR, FATAL = (10000 * (x + 1) for x in range(6)) # @UndefinedVariable +OTHER, DEBUG, INFO, WARN, ERROR, FATAL = ( + 10000 * (x + 1) for x in range(6) +) # @UndefinedVariable # Logging sources -INTERNAL, ACTOR, TRANSPORT, OSMANAGER, UNKNOWN, WEB, ADMIN, SERVICE = ('internal', 'actor', 'transport', 'osmanager', 'unknown', 'web', 'admin', 'service') +INTERNAL, ACTOR, TRANSPORT, OSMANAGER, UNKNOWN, WEB, ADMIN, SERVICE = ( + 'internal', + 'actor', + 'transport', + 'osmanager', + 'unknown', + 'web', + 'admin', + 'service', +) -OTHERSTR, DEBUGSTR, INFOSTR, WARNSTR, ERRORSTR, FATALSTR = ('OTHER', 'DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL') +OTHERSTR, DEBUGSTR, INFOSTR, WARNSTR, ERRORSTR, FATALSTR = ( + 'OTHER', + 'DEBUG', + 'INFO', + 'WARN', + 'ERROR', + 'FATAL', +) # Names for defined log levels __nameLevels = { @@ -57,7 +75,7 @@ __nameLevels = { WARNSTR: WARN, ERRORSTR: ERROR, FATALSTR: FATAL, - OTHERSTR: OTHER + OTHERSTR: OTHER, } # Reverse dict of names @@ -76,15 +94,15 @@ def logStrFromLevel(level: int) -> str: def useLog( - type_: str, - serviceUniqueId: str, - serviceIp: str, - username: str, - srcIP: typing.Optional[str] = None, - srcUser: typing.Optional[str] = None, - userServiceName: typing.Optional[str] = None, - poolName: typing.Optional[str] = None - ) -> None: + type_: str, + serviceUniqueId: str, + serviceIp: str, + username: str, + srcIP: typing.Optional[str] = None, + srcUser: typing.Optional[str] = None, + userServiceName: typing.Optional[str] = None, + poolName: typing.Optional[str] = None, +) -> None: """ Logs an "use service" event (logged from actors) :param type_: Type of event (commonly 'login' or 'logout') @@ -99,21 +117,36 @@ def useLog( userServiceName = 'unknown' if userServiceName is None else userServiceName poolName = 'unknown' if poolName is None else poolName - useLogger.info('|'.join([type_, serviceUniqueId, serviceIp, srcIP, srcUser, username, userServiceName, poolName])) + useLogger.info( + '|'.join( + [ + type_, + serviceUniqueId, + serviceIp, + srcIP, + srcUser, + username, + userServiceName, + poolName, + ] + ) + ) def doLog( - wichObject: 'Model', - level: int, - message: str, - source: str = UNKNOWN, - avoidDuplicates: bool = True - ) -> None: + wichObject: 'Model', + level: int, + message: str, + source: str = UNKNOWN, + avoidDuplicates: bool = True, +) -> None: logger.debug('%s %s %s', wichObject, level, message) logManager().doLog(wichObject, level, message, source, avoidDuplicates) -def getLogs(wichObject: 'Model', limit: typing.Optional[int] = None) -> typing.List[typing.Dict]: +def getLogs( + wichObject: 'Model', limit: typing.Optional[int] = None +) -> typing.List[typing.Dict]: """ Get the logs associated with "wichObject", limiting to "limit" (default is GlobalConfig.MAX_LOGS_PER_ELEMENT) """ diff --git a/server/src/uds/core/util/model.py b/server/src/uds/core/util/model.py index 2595fdc3c..f738a58c0 100644 --- a/server/src/uds/core/util/model.py +++ b/server/src/uds/core/util/model.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2014-2020 Virtual Cable S.L. +# Copyright (c) 2014-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # diff --git a/server/src/uds/core/util/modfinder.py b/server/src/uds/core/util/modfinder.py index ce24a3621..8ef8ba333 100644 --- a/server/src/uds/core/util/modfinder.py +++ b/server/src/uds/core/util/modfinder.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -43,10 +43,10 @@ import uds.dispatchers # pylint: disable=unused-import logger = logging.getLogger(__name__) -patterns: typing.List = [] +patterns: typing.List[typing.Any] = [] -def loadModulesUrls(): +def loadModulesUrls() -> typing.List[typing.Any]: logger.debug('Looking for dispatching modules') if not patterns: logger.debug('Looking for patterns') @@ -56,11 +56,11 @@ def loadModulesUrls(): for _, name, _ in pkgutil.iter_modules([pkgpath]): fullModName = '{}.{}.urls'.format(modName, name) try: - # mod = __import__(fullModName, globals(), locals(), ['urlpatterns'], 0) mod = importlib.import_module(fullModName) - logger.debug('Loaded mod %s, url %s', mod, mod.urlpatterns) + urlpatterns: typing.List[typing.Any] = getattr(mod, 'urlpatterns') + logger.debug('Loaded mod %s, url %s', mod, urlpatterns) # Append patters from mod - for up in mod.urlpatterns: + for up in urlpatterns: patterns.append(up) except Exception: logger.exception('Loading patterns') diff --git a/server/src/uds/core/util/net.py b/server/src/uds/core/util/net.py index 49feeac9a..8cb54631f 100644 --- a/server/src/uds/core/util/net.py +++ b/server/src/uds/core/util/net.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2012 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -89,7 +89,6 @@ def networkFromString(strNets: str) -> NetworkType: return typing.cast(NetworkType, networksFromString(strNets, False)) -# pylint: disable=too-many-return-statements, too-many-branches, too-many-statements def networksFromString( strNets: str, allowMultipleNetworks: bool = True ) -> typing.Union[NetworklistType, NetworkType]: @@ -109,12 +108,12 @@ def networksFromString( inputString = strNets logger.debug('Getting network from %s', strNets) - def check(*args): + def check(*args) -> None: for n in args: if int(n) < 0 or int(n) > 255: raise Exception() - def toNum(*args): + def toNum(*args) -> int: start = 256 * 256 * 256 val = 0 for n in args: @@ -122,7 +121,7 @@ def networksFromString( start >>= 8 return val - def maskFromBits(nBits): + def maskFromBits(nBits: int) -> int: v = 0 for n in range(nBits): v |= 1 << (31 - n) diff --git a/server/src/uds/core/util/os_detector.py b/server/src/uds/core/util/os_detector.py index 10410ea57..24e4ac5d0 100644 --- a/server/src/uds/core/util/os_detector.py +++ b/server/src/uds/core/util/os_detector.py @@ -156,12 +156,3 @@ def getOsFromUA( res.Version = match.groups(1)[0] # type: ignore return res - - -def getOsFromRequest(request): - try: - return request.os - except Exception: - request.os = getOsFromUA(request.META.get('HTTP_USER_AGENT')) - - return request.os diff --git a/server/src/uds/core/util/permissions.py b/server/src/uds/core/util/permissions.py index 620b7a0bc..42f0ea51d 100644 --- a/server/src/uds/core/util/permissions.py +++ b/server/src/uds/core/util/permissions.py @@ -54,10 +54,16 @@ def clean(obj: 'Model') -> None: def getPermissions(obj: 'Model') -> typing.List[models.Permissions]: - return list(models.Permissions.enumeratePermissions(object_type=ot.getObjectType(obj), object_id=obj.pk)) + return list( + models.Permissions.enumeratePermissions( + object_type=ot.getObjectType(obj), object_id=obj.pk + ) + ) -def getEffectivePermission(user: 'models.User', obj: 'Model', root: bool = False) -> int: +def getEffectivePermission( + user: 'models.User', obj: 'Model', root: bool = False +) -> int: try: if user.is_admin: return PERMISSION_ALL @@ -68,23 +74,49 @@ def getEffectivePermission(user: 'models.User', obj: 'Model', root: bool = False # Just check permissions for staff members # root means for "object type" not for an object if root is False: - return models.Permissions.getPermissions(user=user, groups=user.groups.all(), object_type=ot.getObjectType(obj), object_id=obj.pk) + return models.Permissions.getPermissions( + user=user, + groups=user.groups.all(), + object_type=ot.getObjectType(obj), + object_id=obj.pk, + ) - return models.Permissions.getPermissions(user=user, groups=user.groups.all(), object_type=ot.getObjectType(obj)) + return models.Permissions.getPermissions( + user=user, groups=user.groups.all(), object_type=ot.getObjectType(obj) + ) except Exception: return PERMISSION_NONE -def addUserPermission(user: 'models.User', obj: 'Model', permission: int = PERMISSION_READ): +def addUserPermission( + user: 'models.User', obj: 'Model', permission: int = PERMISSION_READ +): # Some permissions added to some object types needs at least READ_PERMISSION on parent - models.Permissions.addPermission(user=user, object_type=ot.getObjectType(obj), object_id=obj.pk, permission=permission) + models.Permissions.addPermission( + user=user, + object_type=ot.getObjectType(obj), + object_id=obj.pk, + permission=permission, + ) -def addGroupPermission(group: 'models.Group', obj: 'Model', permission: int = PERMISSION_READ): - models.Permissions.addPermission(group=group, object_type=ot.getObjectType(obj), object_id=obj.pk, permission=permission) +def addGroupPermission( + group: 'models.Group', obj: 'Model', permission: int = PERMISSION_READ +): + models.Permissions.addPermission( + group=group, + object_type=ot.getObjectType(obj), + object_id=obj.pk, + permission=permission, + ) -def checkPermissions(user: 'models.User', obj: 'Model', permission: int = PERMISSION_ALL, root: bool = False): +def checkPermissions( + user: 'models.User', + obj: 'Model', + permission: int = PERMISSION_ALL, + root: bool = False, +): return getEffectivePermission(user, obj, root) >= permission diff --git a/server/src/uds/core/util/request.py b/server/src/uds/core/util/request.py index 578c5d3fa..a175e4957 100644 --- a/server/src/uds/core/util/request.py +++ b/server/src/uds/core/util/request.py @@ -49,7 +49,7 @@ class ExtendedHttpRequest(HttpRequest): ip: str ip_proxy: str os: DictAsObj - user: typing.Optional[User] # type: ignore # HttpRequests users "user" for it own, but we redefine it because base is not used... + user: typing.Optional[User] # type: ignore class ExtendedHttpRequestWithUser(ExtendedHttpRequest): @@ -72,6 +72,7 @@ def getRequest() -> ExtendedHttpRequest: return ExtendedHttpRequest() + def delCurrentRequest() -> None: ident = getIdent() logger.debug('Deleting %s', ident) @@ -83,6 +84,7 @@ def delCurrentRequest() -> None: except Exception: logger.exception('Deleting stored request') + def cleanOldRequests() -> None: logger.debug('Cleaning stuck requests from %s', _requests) # No request lives 60 seconds, so 60 seconds is fine @@ -98,7 +100,7 @@ def cleanOldRequests() -> None: del _requests[ident] except Exception: pass # Ignore it silently - + def setRequest(request: ExtendedHttpRequest): _requests[getIdent()] = ( diff --git a/server/src/uds/core/util/state.py b/server/src/uds/core/util/state.py index 74d3e17b4..faadbcb29 100644 --- a/server/src/uds/core/util/state.py +++ b/server/src/uds/core/util/state.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -39,6 +39,7 @@ class State: This class represents possible states for objects at database. Take in consideration that objects do not have to support all states, they are here for commodity """ + ACTIVE = 'A' BLOCKED = 'B' CANCELED = 'C' @@ -58,7 +59,7 @@ class State: RUNNING = 'W' FOR_EXECUTE = 'X' MAINTENANCE = 'Y' # "Visual" state, no element will be in fact in maintenance, but used to show "Services Pools" for which a Provider is in maintenance - WAITING_OS = 'Z' # "Visual" state, no element will be in fact in WAITING_OS, but used to show "User Services" that are waiting for os manager + WAITING_OS = 'Z' # "Visual" state, no element will be in fact in WAITING_OS, but used to show "User Services" that are waiting for os manager META_MEMBER = 'V' # "Visual" state, no element will be in fact in WAITING_OS, but used to show "User Services" that are waiting for os manager string = { @@ -68,7 +69,7 @@ class State: LAUNCHING: _('Waiting publication'), PREPARING: _('In preparation'), USABLE: _('Valid'), - REMOVABLE: _('Removing'), # Display as it is removing + REMOVABLE: _('Removing'), # Display as it is removing RESTRAINED: _('Restrained'), REMOVING: _('Removing'), REMOVED: _('Removed'), @@ -82,7 +83,7 @@ class State: MAINTENANCE: _('In maintenance'), WAITING_OS: _('Waiting OS'), SLOWED_DOWN: _('Too many preparing services'), - META_MEMBER: _('Meta member') + META_MEMBER: _('Meta member'), } # States that are merely for "information" to the user. They don't contain any usable instance diff --git a/server/src/uds/core/util/state_queue.py b/server/src/uds/core/util/state_queue.py index cf0d89743..0387c625b 100644 --- a/server/src/uds/core/util/state_queue.py +++ b/server/src/uds/core/util/state_queue.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -32,6 +32,7 @@ """ import typing + class StateQueue: _queue: typing.List[typing.Any] _current: typing.Optional[typing.Any] @@ -41,7 +42,10 @@ class StateQueue: self._current = None def __str__(self): - res = '' % (self._current, ','.join(state for state in self._queue)) + res = '' % ( + self._current, + ','.join(state for state in self._queue), + ) return res def clearQueue(self) -> None: @@ -60,10 +64,11 @@ class StateQueue: def contains(self, state: typing.Any) -> bool: # if self._queue.co - for s in self._queue: - if s == state: - return True - return False + return state in self._queue + # for s in self._queue: + # if s == state: + # return True + # return False def push_back(self, state: typing.Any) -> None: self._queue.append(state) diff --git a/server/src/uds/core/util/storage.py b/server/src/uds/core/util/storage.py index cef4d1811..944406591 100644 --- a/server/src/uds/core/util/storage.py +++ b/server/src/uds/core/util/storage.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2020 Virtual Cable S.L.U. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -97,7 +97,7 @@ class StorageAsDict(MutableMapping): owner (str): owner of the storage group (typing.Optional[str]): group for this dict atomic (bool, optional): if True, operations with DB will be atomic - compat (bool, optional): if True, keys will be SAVED with old format + compat (bool, optional): if True, keys will be SAVED with old format (that is, without the key) so it can be read by old api """ self._group = group or '' diff --git a/server/src/uds/core/util/thread_pool.py b/server/src/uds/core/util/thread_pool.py deleted file mode 100644 index 8c8c5e64d..000000000 --- a/server/src/uds/core/util/thread_pool.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- - -# -# Copyright (c) 2013 Virtual Cable S.L. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from threading import Thread -import logging -import queue - -logger = logging.getLogger(__name__) - -DEFAULT_QUEUE_SIZE = 32 - - -class Worker(Thread): - def __init__(self, tasks): - Thread.__init__(self) - self._tasks = tasks - self._stop = False - self.start() - - def notifyStop(self): - self._stop = True - - def run(self): - while self._stop is False: - try: - func, args, kargs = self._tasks.get(block=True, timeout=1) - except queue.Empty: - continue - - try: - func(*args, **kargs) - except Exception: - logger.exception('ThreadPool Worker') - - self._tasks.task_done() - - -class ThreadPool: - def __init__(self, num_threads, queueSize=DEFAULT_QUEUE_SIZE): - self._tasks = queue.Queue(queueSize) - self._numThreads = num_threads - self._threads = [] - - def add_task(self, func, *args, **kargs): - """ - Add a task to the queue - """ - if not self._threads: - for _ in range(self._numThreads): - self._threads.append(Worker(self._tasks)) - - self._tasks.put((func, args, kargs)) - - def wait_completion(self): - """ - Wait for completion of all the tasks in the queue - """ - self._tasks.join() - - # Now we will close all running tasks - # In case new tasks are inserted after using this, new threads will be created - # to handle tasks - for n in self._threads: - n.notifyStop() - - for n in self._threads: - n.join() - - self._threads = [] diff --git a/server/src/uds/core/util/tools.py b/server/src/uds/core/util/tools.py index 9bc8eeb66..88934ce1b 100644 --- a/server/src/uds/core/util/tools.py +++ b/server/src/uds/core/util/tools.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -43,13 +43,16 @@ import django.template.defaultfilters as filters from uds.core import services + class DictAsObj(dict): """ Returns a mix between a dict and an obj Can be accesses as .xxxx or ['xxx'] """ - def __init__(self, dct: typing.Optional[typing.Dict[str, typing.Any]] = None, **kwargs): + def __init__( + self, dct: typing.Optional[typing.Dict[str, typing.Any]] = None, **kwargs + ): if dct: self.__dict__.update(dct) self.__dict__.update(kwargs) @@ -58,14 +61,11 @@ class DictAsObj(dict): return self.__dict__[key] def __unicode__(self): - return ', '.join( - '{}={}'.format(v, self.__dict__[v]) for v in self.__dict__ - ) + return ', '.join('{}={}'.format(v, self.__dict__[v]) for v in self.__dict__) # pylint: disable=protected-access class CaseInsensitiveDict(dict): - @classmethod def _k(cls, key): return key.lower() if isinstance(key, str) else key @@ -87,13 +87,19 @@ class CaseInsensitiveDict(dict): return super(CaseInsensitiveDict, self).__contains__(self.__class__._k(key)) def pop(self, key, *args, **kwargs): - return super(CaseInsensitiveDict, self).pop(self.__class__._k(key), *args, **kwargs) + return super(CaseInsensitiveDict, self).pop( + self.__class__._k(key), *args, **kwargs + ) def get(self, key, *args, **kwargs): - return super(CaseInsensitiveDict, self).get(self.__class__._k(key), *args, **kwargs) + return super(CaseInsensitiveDict, self).get( + self.__class__._k(key), *args, **kwargs + ) def setdefault(self, key, *args, **kwargs): - return super(CaseInsensitiveDict, self).setdefault(self.__class__._k(key), *args, **kwargs) + return super(CaseInsensitiveDict, self).setdefault( + self.__class__._k(key), *args, **kwargs + ) def update(self, E=None, **F): if E is None: @@ -106,6 +112,7 @@ class CaseInsensitiveDict(dict): v = super(CaseInsensitiveDict, self).pop(k) self.__setitem__(k, v) + def asList(value: typing.Any) -> typing.List[typing.Any]: if isinstance(value, list): return value @@ -116,6 +123,7 @@ def asList(value: typing.Any) -> typing.List[typing.Any]: except Exception: return [value] + def packageRelativeFile(moduleName: str, fileName: str) -> str: """ Helper to get image path from relative to a module. @@ -143,8 +151,9 @@ def secondsToTimeString(seconds: int) -> str: hours %= 24 return ugettext('{} days {:d}:{:02d}:{:02d}').format(days, hours, minutes, seconds) + def checkValidBasename(baseName: str, length: int = -1) -> None: - """"Checks if the basename + length is valid for services. Raises an exception if not valid" + """ "Checks if the basename + length is valid for services. Raises an exception if not valid" Arguments: baseName {str} -- basename to check @@ -158,16 +167,25 @@ def checkValidBasename(baseName: str, length: int = -1) -> None: None -- [description] """ if re.match(r'^[a-zA-Z0-9][a-zA-Z0-9-]*$', baseName) is None: - raise services.Service.ValidationException(ugettext('The basename is not a valid for a hostname')) + raise services.Service.ValidationException( + ugettext('The basename is not a valid for a hostname') + ) if length == 0: - raise services.Service.ValidationException(ugettext('The length of basename plus length must be greater than 0')) + raise services.Service.ValidationException( + ugettext('The length of basename plus length must be greater than 0') + ) if length != -1 and len(baseName) + length > 15: - raise services.Service.ValidationException(ugettext('The length of basename plus length must not be greater than 15')) + raise services.Service.ValidationException( + ugettext('The length of basename plus length must not be greater than 15') + ) if baseName.isdigit(): - raise services.Service.ValidationException(ugettext('The machine name can\'t be only numbers')) + raise services.Service.ValidationException( + ugettext('The machine name can\'t be only numbers') + ) + def removeControlCharacters(s: str) -> str: - return ''.join(ch for ch in s if unicodedata.category(ch)[0]!="C") + return ''.join(ch for ch in s if unicodedata.category(ch)[0] != "C") diff --git a/server/src/uds/core/util/unique.py b/server/src/uds/core/util/unique.py index 196d8934c..bd420cfb4 100644 --- a/server/src/uds/core/util/unique.py +++ b/server/src/uds/core/util/unique.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2012-2019 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +11,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -30,10 +29,6 @@ """ @author: Adolfo Gómez, dkmaster at dkmon dot com """ -# Helper to acommodate all unique generators in one place -# pylint: disable=unused-import - from .unique_gid_generator import UniqueGIDGenerator from .unique_mac_generator import UniqueMacGenerator from .unique_name_generator import UniqueNameGenerator - diff --git a/server/src/uds/core/util/unique_gid_generator.py b/server/src/uds/core/util/unique_gid_generator.py index 06cdd40a9..17a029c1c 100644 --- a/server/src/uds/core/util/unique_gid_generator.py +++ b/server/src/uds/core/util/unique_gid_generator.py @@ -38,7 +38,6 @@ logger = logging.getLogger(__name__) class UniqueGIDGenerator(UniqueIDGenerator): - def __init__(self, owner, baseName=None): super().__init__('id', owner, baseName) diff --git a/server/src/uds/core/util/unique_id_generator.py b/server/src/uds/core/util/unique_id_generator.py index bdcc87f9e..f4e0ee82a 100644 --- a/server/src/uds/core/util/unique_id_generator.py +++ b/server/src/uds/core/util/unique_id_generator.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -54,18 +54,24 @@ class UniqueIDGenerator: _owner: str _baseName: str - def __init__(self, typeName: str, owner: str, baseName: typing.Optional[str] = None): + def __init__( + self, typeName: str, owner: str, baseName: typing.Optional[str] = None + ): self._owner = owner + typeName self._baseName = 'uds' if baseName is None else baseName def setBaseName(self, newBaseName: str): self._baseName = newBaseName - def __filter(self, rangeStart: int, rangeEnd: int = MAX_SEQ, forUpdate: bool = False) -> QuerySet: + def __filter( + self, rangeStart: int, rangeEnd: int = MAX_SEQ, forUpdate: bool = False + ) -> QuerySet: # Order is defined on UniqueId model, and is '-seq' by default (so this gets items in sequence order) # if not for update, do not use the clause :) obj = UniqueId.objects.select_for_update() if forUpdate else UniqueId.objects - return obj.filter(basename=self._baseName, seq__gte=rangeStart, seq__lte=rangeEnd) # @UndefinedVariable + return obj.filter( + basename=self._baseName, seq__gte=rangeStart, seq__lte=rangeEnd + ) # @UndefinedVariable def get(self, rangeStart: int = 0, rangeEnd: int = MAX_SEQ) -> int: """ @@ -99,7 +105,9 @@ class UniqueIDGenerator: if not item: # logger.debug('No free found, creating new one') try: - last = flt.filter(assigned=True)[0] # DB Returns correct order so the 0 item is the last + last = flt.filter(assigned=True)[ + 0 + ] # DB Returns correct order so the 0 item is the last seq = last.seq + 1 except IndexError: # If there is no assigned at database seq = rangeStart @@ -108,7 +116,13 @@ class UniqueIDGenerator: return -1 # No ids free in range # May ocurr on some circustance that a concurrency access gives same item twice, in this case, we # will get an "duplicate key error", - UniqueId.objects.create(owner=self._owner, basename=self._baseName, seq=seq, assigned=True, stamp=stamp) # @UndefinedVariable + UniqueId.objects.create( + owner=self._owner, + basename=self._baseName, + seq=seq, + assigned=True, + stamp=stamp, + ) # @UndefinedVariable break except OperationalError: # Locked, may ocurr for example on sqlite. We will wait a bit # logger.exception('Got database locked') @@ -125,24 +139,20 @@ class UniqueIDGenerator: return seq def transfer(self, seq: int, toUidGen: 'UniqueIDGenerator') -> bool: - self.__filter( - 0, forUpdate=True - ).filter( - owner=self._owner, seq=seq - ).update( - owner=toUidGen._owner, basename=toUidGen._baseName, stamp=getSqlDatetimeAsUnix() # pylint: disable=protected-access + self.__filter(0, forUpdate=True).filter(owner=self._owner, seq=seq).update( + owner=toUidGen._owner, + basename=toUidGen._baseName, + stamp=getSqlDatetimeAsUnix(), # pylint: disable=protected-access ) return True def free(self, seq) -> None: logger.debug('Freeing seq %s from %s (%s)', seq, self._owner, self._baseName) with transaction.atomic(): - flt = self.__filter( - 0, forUpdate=True - ).filter( - owner=self._owner, seq=seq - ).update( - owner='', assigned=False, stamp=getSqlDatetimeAsUnix() + flt = ( + self.__filter(0, forUpdate=True) + .filter(owner=self._owner, seq=seq) + .update(owner='', assigned=False, stamp=getSqlDatetimeAsUnix()) ) if flt > 0: self.__purge() @@ -157,13 +167,21 @@ class UniqueIDGenerator: # logger.exception('Error here') seq = 0 with transaction.atomic(): - self.__filter(seq).delete() # Clean ups all unassigned after last assigned in this range + self.__filter( + seq + ).delete() # Clean ups all unassigned after last assigned in this range def release(self) -> None: - UniqueId.objects.select_for_update().filter(owner=self._owner).update(assigned=False, owner='', stamp=getSqlDatetimeAsUnix()) # @UndefinedVariable + UniqueId.objects.select_for_update().filter(owner=self._owner).update( + assigned=False, owner='', stamp=getSqlDatetimeAsUnix() + ) # @UndefinedVariable self.__purge() def releaseOlderThan(self, stamp=None) -> None: stamp = getSqlDatetimeAsUnix() if stamp is None else stamp - UniqueId.objects.select_for_update().filter(owner=self._owner, stamp__lt=stamp).update(assigned=False, owner='', stamp=stamp) # @UndefinedVariable + UniqueId.objects.select_for_update().filter( + owner=self._owner, stamp__lt=stamp + ).update( + assigned=False, owner='', stamp=stamp + ) # @UndefinedVariable self.__purge() diff --git a/server/src/uds/core/util/unique_mac_generator.py b/server/src/uds/core/util/unique_mac_generator.py index d76897662..ea4c1c97e 100644 --- a/server/src/uds/core/util/unique_mac_generator.py +++ b/server/src/uds/core/util/unique_mac_generator.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -39,8 +39,7 @@ logger = logging.getLogger(__name__) class UniqueMacGenerator(UniqueIDGenerator): - - def __init__(self, owner: str): + def __init__(self, owner: str) -> None: super().__init__('mac', owner, '\tmac') def __toInt(self, mac: str) -> int: @@ -52,14 +51,14 @@ class UniqueMacGenerator(UniqueIDGenerator): return re.sub(r"(..)", r"\1:", "%0*X" % (12, seq))[:-1] # noinspection PyMethodOverriding - def get(self, macRange: str) -> str: # type: ignore # pylint: disable=arguments-differ + def get(self, macRange: str) -> str: # type: ignore firstMac, lastMac = macRange.split('-') return self.__toMac(super().get(self.__toInt(firstMac), self.__toInt(lastMac))) - def transfer(self, mac: str, toUMgen: 'UniqueMacGenerator'): # type: ignore # pylint: disable=arguments-differ + def transfer(self, mac: str, toUMgen: 'UniqueMacGenerator'): # type: ignore super().transfer(self.__toInt(mac), toUMgen) - def free(self, mac: str): # pylint: disable=arguments-differ + def free(self, mac: str) -> None: super().free(self.__toInt(mac)) # Release is inherited, no mod needed diff --git a/server/src/uds/core/util/unique_name_generator.py b/server/src/uds/core/util/unique_name_generator.py index ff011768d..325dc9b42 100644 --- a/server/src/uds/core/util/unique_name_generator.py +++ b/server/src/uds/core/util/unique_name_generator.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -55,8 +55,8 @@ class UniqueNameGenerator(UniqueIDGenerator): def transfer(self, baseName: str, name: str, toUNGen: 'UniqueNameGenerator'): # type: ignore # pylint: disable=arguments-differ self.setBaseName(baseName) - super().transfer(int(name[len(self._baseName):]), toUNGen) + super().transfer(int(name[len(self._baseName) :]), toUNGen) def free(self, baseName: str, name: str) -> None: # type: ignore # pylint: disable=arguments-differ self.setBaseName(baseName) - super().free(int(name[len(self._baseName):])) + super().free(int(name[len(self._baseName) :])) diff --git a/server/src/uds/core/util/validators.py b/server/src/uds/core/util/validators.py index f83808751..fb6f6b289 100644 --- a/server/src/uds/core/util/validators.py +++ b/server/src/uds/core/util/validators.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2014-2019 Virtual Cable S.L. +# Copyright (c) 2014-2019 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -12,7 +12,7 @@ # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -39,13 +39,13 @@ from uds.core.module import Module logger = logging.getLogger(__name__) + def validateNumeric( - numericStr: str, - minValue: typing.Optional[int] = None, - maxValue: typing.Optional[int] = None, - returnAsInteger: bool = True, - fieldName: typing.Optional[str] = None - ) -> typing.Union[str, int]: + value: str, + minValue: typing.Optional[int] = None, + maxValue: typing.Optional[int] = None, + fieldName: typing.Optional[str] = None, +) -> int: """ Validates that a numeric value is valid :param numericStr: Numeric value to check (as string) @@ -55,41 +55,53 @@ def validateNumeric( :param fieldName: If present, the name of the field for "Raising" exceptions, defaults to "Numeric value" :return: Raises Module.Validation exception if is invalid, else return the value "fixed" """ - numericStr = numericStr.replace(' ', '') + value = value.replace(' ', '') fieldName = fieldName or _('Numeric') try: - numeric = int(numericStr) + numeric = int(value) if minValue is not None and numeric < minValue: - raise Module.ValidationException(_('{0} must be greater than or equal to {1}'.format(fieldName, minValue))) + raise Module.ValidationException( + _( + '{0} must be greater than or equal to {1}'.format( + fieldName, minValue + ) + ) + ) if maxValue is not None and numeric > maxValue: - raise Module.ValidationException(_('{0} must be lower than or equal to {1}'.format(fieldName, maxValue))) + raise Module.ValidationException( + _('{0} must be lower than or equal to {1}'.format(fieldName, maxValue)) + ) - numericStr = str(numeric) + value = str(numeric) except ValueError: - raise Module.ValidationException(_('{0} contains invalid characters').format(fieldName)) + raise Module.ValidationException( + _('{0} contains invalid characters').format(fieldName) + ) - if returnAsInteger: - return int(numericStr) + return int(value) - return numericStr def validateHostname(hostname: str, maxLength: int, asPattern: bool) -> str: if len(hostname) > maxLength: - raise Module.ValidationException(_('{} exceeds maximum host name length.').format(hostname)) + raise Module.ValidationException( + _('{} exceeds maximum host name length.').format(hostname) + ) if hostname[-1] == ".": - hostname = hostname[:-1] # strip exactly one dot from the right, if present + hostname = hostname[:-1] # strip exactly one dot from the right, if present if asPattern: - allowed = re.compile("(?!-)[A-Z\d-]{1,63}$", re.IGNORECASE) + allowed = re.compile(r'(?!-)[A-Z\d-]{1,63}$', re.IGNORECASE) else: - allowed = re.compile("(?!-)[A-Z\d-]{1,63}(? int: :param returnAsInteger: if True, returns value as integer, if not, as string :return: Raises Module.Validation exception if is invalid, else return the value "fixed" """ - return typing.cast(int, validateNumeric(portStr, minValue=0, maxValue=65535, returnAsInteger=True, fieldName='Port')) + return validateNumeric(portStr, minValue=0, maxValue=65535, fieldName='Port') -def validateTimeout(timeOutStr) -> int: +def validateTimeout(timeOutStr: str) -> int: """ Validates that a timeout value is valid :param timeOutStr: timeout to validate :param returnAsInteger: if True, returns value as integer, if not, as string :return: Raises Module.Validation exception if is invalid, else return the value "fixed" """ - return typing.cast(int, validateNumeric(timeOutStr, minValue=0, returnAsInteger=True, fieldName='Timeout')) + return validateNumeric(timeOutStr, minValue=0, fieldName='Timeout') def validateMacRange(macRange: str) -> str: @@ -123,7 +135,9 @@ def validateMacRange(macRange: str) -> str: # Removes white spaces and all to uppercase macRange = macRange.upper().replace(' ', '') - macRE = re.compile(r'^([0-9A-F]{2}[:]){5}([0-9A-F]{2})$') # In fact, it could be XX-XX-XX-XX-XX-XX, but we use - as range separator + macRE = re.compile( + r'^([0-9A-F]{2}[:]){5}([0-9A-F]{2})$' + ) # In fact, it could be XX-XX-XX-XX-XX-XX, but we use - as range separator try: macRangeStart, macRangeEnd = macRange.split('-') @@ -133,6 +147,10 @@ def validateMacRange(macRange: str) -> str: if macRangeStart > macRangeEnd: raise Exception() except Exception: - raise Module.ValidationException(_('Invalid mac range. Mac range must be in format XX:XX:XX:XX:XX:XX-XX:XX:XX:XX:XX:XX')) + raise Module.ValidationException( + _( + 'Invalid mac range. Mac range must be in format XX:XX:XX:XX:XX:XX-XX:XX:XX:XX:XX:XX' + ) + ) return macRange diff --git a/server/src/uds/core/util/xml2dict.py b/server/src/uds/core/util/xml2dict.py index cc9720e6e..aa9b0ded1 100644 --- a/server/src/uds/core/util/xml2dict.py +++ b/server/src/uds/core/util/xml2dict.py @@ -1,12 +1,7 @@ -""" -Created on Jul 11, 2016 - -@author: dkmaster -""" # -*- coding: utf-8 -*- # -# Copyright (c) 2013-2019 Virtual Cable S.L. +# Copyright (c) 2013-2021 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -17,7 +12,7 @@ Created on Jul 11, 2016 # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. -# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # @@ -31,20 +26,27 @@ Created on Jul 11, 2016 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" +@author: dkmaster@dkmon.com +""" import typing from collections import defaultdict from xml.etree import cElementTree + def etree_to_dict(t): - d = {t.tag: {} if t.attrib else None} + d = {} + if t.attrib: + d.update({t.tag: {}}) + children = list(t) if children: dd = defaultdict(list) for dc in map(etree_to_dict, children): for k, v in dc.items(): dd[k].append(v) - d = {t.tag: {k:v[0] if len(v) == 1 else v for k, v in dd.items()}} + d = {t.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}} if t.attrib: d[t.tag].update(('@' + k, v) for k, v in t.attrib.items()) if t.text: @@ -56,5 +58,6 @@ def etree_to_dict(t): d[t.tag] = text return d + def parse(xml_string: str) -> typing.Dict: - return etree_to_dict(cElementTree.XML(xml_string)) + return etree_to_dict(cElementTree.XML(xml_string))