Adding generic importers to simplify module loading

server/src/uds/auths/__init__.py

@@ -40,7 +40,7 @@ The registration of modules is done locating subclases of :py:class:`uds.core.au
 
 .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
 """
-from uds.core.util.modfinder import dynamicLoadAndRegisterModules
+from uds.core.util import modfinder
 
 
 def __loadModules__():
@@ -50,7 +50,7 @@ def __loadModules__():
     """
     from uds.core import auths
 
-    dynamicLoadAndRegisterModules(auths.factory(), auths.Authenticator, __name__)
+    modfinder.dynamicLoadAndRegisterModules(auths.factory(), auths.Authenticator, __name__)
 
 
 __loadModules__()

server/src/uds/core/managers/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 #
-# Copyright (c) 2012-2019 Virtual Cable S.L.
+# Copyright (c) 2012-2022 Virtual Cable S.L.U.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -12,7 +12,7 @@
 #    * Redistributions in binary form must reproduce the above copyright notice,
 #      this list of conditions and the following disclaimer in the documentation
 #      and/or other materials provided with the distribution.
-#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#    * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
 #      may be used to endorse or promote products derived from this software
 #      without specific prior written permission.
 #
@@ -54,33 +54,33 @@ def cryptoManager() -> 'CryptoManager':
 def taskManager() -> 'TaskManager':
     from .task import TaskManager
 
-    return TaskManager.manager()
+    return TaskManager()
 
 
 def downloadsManager() -> 'DownloadsManager':
     from .downloads import DownloadsManager
 
-    return DownloadsManager.manager()
+    return DownloadsManager()
 
 
 def logManager() -> 'LogManager':
     from .log import LogManager
 
-    return LogManager.manager()
+    return LogManager()
 
 
 def userServiceManager() -> 'UserServiceManager':
     from .user_service import UserServiceManager
 
-    return UserServiceManager.manager()
+    return UserServiceManager()
 
 
 def publicationManager() -> 'PublicationManager':
     from .publication import PublicationManager
 
-    return PublicationManager.manager()
+    return PublicationManager()
 
 def notificationsManager() -> 'NotificationsManager':
     from .notifications import NotificationsManager
 
-    return NotificationsManager.manager()
+    return NotificationsManager()

server/src/uds/core/module.py

@@ -97,6 +97,9 @@ class Module(UserInterface, Environmentable, Serializable):
     Environmentable is a base class that provides utility method to access a separate Environment for every single
     module.
     """
+    # Import variable indicating this module is a base class not a real module
+    # Note that Module is not a real module, but a base class for all modules so isBase is not used on this class
+    isBase: typing.ClassVar[bool] = False
 
     # Types
     ValuesType = typing.Optional[

server/src/uds/core/util/factory.py

@@ -30,7 +30,7 @@ class ModuleFactory(typing.Generic[T], metaclass=singleton.Singleton):
         '''
         Inserts an object into the factory.
         '''
-        logger.debug('Adding %s as %s', type_.type(), type_)
+        # logger.debug('Adding %s as %s', type_.type(), type_.__module__)
         typeName = type_.type().lower()
 
         if typeName in self.__objects:

server/src/uds/core/util/modfinder.py

@@ -43,6 +43,7 @@ from uds.core import module
 logger = logging.getLogger(__name__)
 
 T = typing.TypeVar('T', bound=module.Module)
+V = typing.TypeVar('V')
 
 patterns: typing.List[typing.Any] = []
 
@@ -77,35 +78,68 @@ def loadModulesUrls() -> typing.List[typing.Any]:
     return patterns
 
 
+def importModules(modName: str) -> None:
+    # Dinamycally import children of this package.
+    pkgpath = os.path.dirname(typing.cast(str, sys.modules[modName].__file__))
+    logger.info('* Importing modules from %s', pkgpath)
+    for _, name, _ in pkgutil.iter_modules([pkgpath]):
+        logger.info('   - Importing module %s.%s ', modName, name)
+        importlib.import_module('.' + name, modName)  # import module
+    logger.info('* Done importing modules from %s', pkgpath)
+
+    importlib.invalidate_caches()
+
+
+def dynamicLoadAndRegisterPackages(
+    adder: typing.Callable[[typing.Type[V]], None],
+    type_: typing.Type[V],
+    modName: str,
+    *,
+    checker: typing.Optional[typing.Callable[[typing.Type[V]], bool]] = None,
+) -> None:
+    '''
+    Loads all packages from a given package that are subclasses of the given type
+    param adder: Function to use to add the objects, must support "insert" method
+    param type_: Type of the objects to load
+    param modName: Name of the package to load
+    param checker: Function to use to check if the class is registrable
+    '''
+    importModules(modName)
+
+    checkFnc = checker or (lambda x: True)
+
+    def process(classes: typing.Iterable[typing.Type]) -> None:
+        cls: typing.Type[V]
+        for cls in classes:
+            clsSubCls = cls.__subclasses__()
+            if clsSubCls:
+                process(clsSubCls)
+                if not checkFnc(cls):
+                    logger.debug('Node is a base, skipping: %s', cls.__module__)
+                    continue
+
+            logger.info('   - Registering %s', cls.__module__)
+            adder(cls)
+
+    logger.info('* Start registering %s', modName)
+    process(type_.__subclasses__())
+    logger.info('* Done Registering %s', modName)
+
+
 def dynamicLoadAndRegisterModules(
     factory: 'ModuleFactory',
     type_: typing.Type[T],
     modName: str,
-    *,
-    justLeafs: bool = False,
 ) -> None:
     '''
     Loads all modules from a given package that are subclasses of the given type
     param factory: Factory to use to create the objects, must support "insert" method
     param type_: Type of the objects to load
     param modName: Name of the package to load
-    param justLeafs: If true, only leafs will be registered on factory
     '''
-    # Dinamycally import children of this package.
+    dynamicLoadAndRegisterPackages(
-    pkgpath = os.path.dirname(typing.cast(str, sys.modules[modName].__file__))
+        factory.insert,
-    for _, name, _ in pkgutil.iter_modules([pkgpath]):
+        type_,
-        importlib.import_module('.' + name, modName)  # import module
+        modName,
-
+        checker=lambda x: not x.isBase
-    importlib.invalidate_caches()
+    )
-
-    def process(classes: typing.Iterable[typing.Type]) -> None:
-        for cls in classes:
-            clsSubCls = cls.__subclasses__()
-            if clsSubCls:
-                process(clsSubCls)
-                if justLeafs:
-                    continue
-
-            factory.insert(cls)
-
-    process(type_.__subclasses__())

server/src/uds/core/workers/__init__.py

@@ -30,13 +30,11 @@
 """
 .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
 """
-import sys
-import os.path
-import pkgutil
-import importlib
 import typing
 import logging
 
+from uds.core.util import modfinder
+
 logger = logging.getLogger(__name__)
 
 
@@ -48,18 +46,13 @@ def initialize() -> None:
     from uds.core import jobs
     from uds.core.managers import taskManager
 
-    # Dinamycally import children of this package.
+    def registerer(cls: typing.Type[jobs.Job]) -> None:
-    pkgpath = os.path.dirname(typing.cast(str, sys.modules[__name__].__file__))
+        if cls.__module__.startswith('uds.core.workers'):
-    for _, name, _ in pkgutil.iter_modules([pkgpath]):
+            logger.debug('Registering job: %s', cls.__module__)
-        logger.debug('Importing worker %s', name)
-        # __import__(name, globals(), locals(), [], 1)
-        importlib.import_module('.' + name, __name__)  # import module
-
-    importlib.invalidate_caches()
-
-    for cls in jobs.Job.__subclasses__():
-        logger.debug('Examining worker %s', cls.__module__)
-        # Limit to autoregister just workers jobs inside this module
-        if cls.__module__[0:16] == 'uds.core.workers':
-            logger.debug('Added worker %s to list', cls.__module__)
             taskManager().registerJob(cls)
+
+    modfinder.dynamicLoadAndRegisterPackages(
+        registerer,
+        jobs.Job,
+        __name__
+    )

server/src/uds/migrations/__init__.py

@@ -1,17 +1 @@
-from __future__ import unicode_literals
+# Removed old south support
-
-SOUTH_ERROR_MESSAGE = """\n
-For South support, customize the SOUTH_MIGRATION_MODULES setting like so:
-
-    SOUTH_MIGRATION_MODULES = {
-        'uds': 'uds.south_migrations',
-    }
-"""
-
-# Ensure the user is not using Django 1.6 or below with South
-try:
-    from django.db import migrations  # noqa
-except ImportError:
-    migrations = None  # Avoid pep error
-    from django.core.exceptions import ImproperlyConfigured
-    raise ImproperlyConfigured(SOUTH_ERROR_MESSAGE)

server/src/uds/notifiers/__init__.py

@@ -43,7 +43,7 @@ The registration of modules is done locating subclases of :py:class:`uds.core.me
 """
 import logging
 
-from uds.core.util.modfinder import dynamicLoadAndRegisterModules
+from uds.core.util import modfinder
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +55,7 @@ def __loadModules():
     from uds.core import messaging
 
     # We need to import all modules that are descendant of this package
-    dynamicLoadAndRegisterModules(messaging.factory(), messaging.Notifier, __name__)
+    modfinder.dynamicLoadAndRegisterModules(messaging.factory(), messaging.Notifier, __name__)
 
 
 __loadModules()

server/src/uds/osmanagers/__init__.py

@@ -40,7 +40,8 @@ The registration of modules is done locating subclases of :py:class:`uds.core.au
 
 .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
 """
-from uds.core.util.modfinder import dynamicLoadAndRegisterModules
+from uds.core.util import modfinder
+
 
 def __loadModules():
     """
@@ -49,7 +50,10 @@ def __loadModules():
 
     from uds.core import osmanagers
 
-    dynamicLoadAndRegisterModules(osmanagers.factory(), osmanagers.OSManager, __name__)
+    # OSManagers registers everything
+    modfinder.dynamicLoadAndRegisterModules(
+        osmanagers.factory(), osmanagers.OSManager, __name__
+    )
 
 
 __loadModules()

server/src/uds/transports/RDP/rdp.py

@@ -53,6 +53,7 @@ class RDPTransport(BaseRDPTransport):
     Provides access via RDP to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     '''
+    isBase = False
 
     typeName = _('RDP')
     typeType = 'RDPTransport'

server/src/uds/transports/RDP/rdp_base.py

@@ -61,6 +61,7 @@ class BaseRDPTransport(transports.Transport):
     Provides access via RDP to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = True
 
     iconFile = 'rdp.png'
     protocol = transports.protocols.RDP

server/src/uds/transports/RDP/rdptunnel.py

@@ -59,6 +59,7 @@ class TRDPTransport(BaseRDPTransport):
     Provides access via RDP to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = False
 
     iconFile = 'rdp-tunnel.png'
     typeName = _('RDP')

server/src/uds/transports/SPICE/spice.py

@@ -51,6 +51,7 @@ class SPICETransport(BaseSpiceTransport):
     Provides access via SPICE to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = False
 
     typeName = _('SPICE')
     typeType = 'SPICETransport'

server/src/uds/transports/SPICE/spice_base.py

@@ -54,6 +54,7 @@ class BaseSpiceTransport(transports.Transport):
     Provides access via SPICE to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = True
 
     iconFile = 'spice.png'
     protocol = protocols.SPICE

server/src/uds/transports/SPICE/spice_tunnel.py

@@ -56,6 +56,7 @@ class TSPICETransport(BaseSpiceTransport):
     """
     Provides access via SPICE to service.
     """
+    isBase = False
 
     iconFile = 'spice-tunnel.png'
     typeName = _('SPICE')

server/src/uds/transports/X2GO/x2go.py

@@ -51,6 +51,7 @@ class X2GOTransport(BaseX2GOTransport):
     Provides access via X2GO to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = False
 
     typeName = _('X2Go')
     typeType = 'X2GOTransport'

server/src/uds/transports/X2GO/x2go_base.py

@@ -60,6 +60,7 @@ class BaseX2GOTransport(transports.Transport):
     Provides access via X2GO to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = True
 
     iconFile = 'x2go.png'
     protocol = transports.protocols.X2GO

server/src/uds/transports/X2GO/x2go_tunnel.py

@@ -29,8 +29,6 @@
 """
 @author: Adolfo Gómez, dkmaster at dkmon dot com
 """
-import random
-import string
 import logging
 import typing
 
@@ -59,6 +57,8 @@ class TX2GOTransport(BaseX2GOTransport):
     Provides access via X2GO to service.
     This transport can use an domain. If username processed by authenticator contains '@', it will split it and left-@-part will be username, and right password
     """
+    isBase = False
+
     iconFile = 'x2go-tunnel.png'
     typeName = _('X2Go')
     typeType = 'TX2GOTransport'

server/src/uds/transports/__init__.py

@@ -41,7 +41,8 @@ The registration of modules is done locating subclases of :py:class:`uds.core.au
 .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
 """
 
-from uds.core.util.modfinder import dynamicLoadAndRegisterModules
+from uds.core.util import modfinder
+
 
 def __init__():
     """
@@ -50,6 +51,10 @@ def __init__():
     """
     from uds.core import transports
 
-    dynamicLoadAndRegisterModules(transports.factory(), transports.Transport, __name__)
+    # Transports ignores non leaf modules
+    modfinder.dynamicLoadAndRegisterModules(
+        transports.factory(), transports.Transport, __name__
+    )
+
 
 __init__()