From 9defc46083fc23fc957eb225e5ad53925eb1b0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20G=C3=B3mez=20Garc=C3=ADa?= Date: Thu, 15 Feb 2024 19:40:36 +0100 Subject: [PATCH] Fixing up fixed_service generalization, and starting a test for this... --- server/src/uds/core/managers/servers.py | 70 ++++----- server/src/uds/core/managers/user_service.py | 44 +++--- server/src/uds/core/services/service.py | 10 +- .../fixed_machine/fixed_service.py | 48 ++++-- .../fixed_machine/fixed_userservice.py | 144 ++++++++++++------ .../uds/services/Proxmox/deployment_fixed.py | 23 +-- .../src/uds/services/Proxmox/service_fixed.py | 6 +- .../core/managers/test_servers_managed.py | 16 +- .../core/managers/test_servers_unmanaged.py | 10 +- server/tests/core/services/__init__.py | 30 ++++ .../core/services/specializations/__init__.py | 0 .../specializations/test_fixedservice.py | 122 +++++++++++++++ 12 files changed, 374 insertions(+), 149 deletions(-) create mode 100644 server/tests/core/services/__init__.py create mode 100644 server/tests/core/services/specializations/__init__.py create mode 100644 server/tests/core/services/specializations/test_fixedservice.py diff --git a/server/src/uds/core/managers/servers.py b/server/src/uds/core/managers/servers.py index 4708b3a17..3cec48035 100644 --- a/server/src/uds/core/managers/servers.py +++ b/server/src/uds/core/managers/servers.py @@ -133,21 +133,21 @@ class ServerManager(metaclass=singleton.Singleton): def _find_best_server( self, - userService: 'models.UserService', - serverGroup: 'models.ServerGroup', + userservice: 'models.UserService', + server_group: 'models.ServerGroup', now: datetime.datetime, - minMemoryMB: int = 0, - excludeServersUUids: typing.Optional[typing.Set[str]] = None, + min_memory_mb: int = 0, + excluded_servers_uuids: typing.Optional[typing.Set[str]] = None, ) -> tuple['models.Server', 'types.servers.ServerStats']: """ Finds the best server for a service """ best: typing.Optional[tuple['models.Server', 'types.servers.ServerStats']] = None unmanaged_list: list['models.Server'] = [] - fltrs = serverGroup.servers.filter(maintenance_mode=False) + fltrs = server_group.servers.filter(maintenance_mode=False) fltrs = fltrs.filter(Q(locked_until=None) | Q(locked_until__lte=now)) # Only unlocked servers - if excludeServersUUids: - fltrs = fltrs.exclude(uuid__in=excludeServersUUids) + if excluded_servers_uuids: + fltrs = fltrs.exclude(uuid__in=excluded_servers_uuids) serversStats = self.get_server_stats(fltrs) @@ -156,7 +156,7 @@ class ServerManager(metaclass=singleton.Singleton): if stats is None: unmanaged_list.append(server) continue - if minMemoryMB and stats.memused // (1024 * 1024) < minMemoryMB: # Stats has minMemory in bytes + if min_memory_mb and stats.memused // (1024 * 1024) < min_memory_mb: # Stats has minMemory in bytes continue if best is None or stats.weight() < best[1].weight(): @@ -184,53 +184,53 @@ class ServerManager(metaclass=singleton.Singleton): # If best was locked, notify it (will be notified again on assign) if best[0].locked_until is not None: - requester.ServerApiRequester(best[0]).notify_release(userService) + requester.ServerApiRequester(best[0]).notify_release(userservice) return best def assign( self, - userService: 'models.UserService', - serverGroup: 'models.ServerGroup', - serviceType: types.services.ServiceType = types.services.ServiceType.VDI, - minMemoryMB: int = 0, # Does not apply to unmanged servers - lockTime: typing.Optional[datetime.timedelta] = None, + userservice: 'models.UserService', + server_group: 'models.ServerGroup', + service_type: types.services.ServiceType = types.services.ServiceType.VDI, + min_memory_mb: int = 0, # Does not apply to unmanged servers + lock_interval: typing.Optional[datetime.timedelta] = None, server: typing.Optional['models.Server'] = None, # If not note - excludeServersUUids: typing.Optional[typing.Set[str]] = None, + excluded_servers_uuids: typing.Optional[typing.Set[str]] = None, ) -> typing.Optional[types.servers.ServerCounter]: """ Select a server for an userservice to be assigned to Args: - userService: User service to assign server to (in fact, user of userservice) and to notify - serverGroup: Server group to select server from - serverType: Type of service to assign - minMemoryMB: Minimum memory required for server in MB, does not apply to unmanaged servers - maxLockTime: If not None, lock server for this time + userservice: User service to assign server to (in fact, user of userservice) and to notify + server_group: Server group to select server from + service_type: Type of service to assign + min_memory_mb: Minimum memory required for server in MB, does not apply to unmanaged servers + lock_interval: If not None, lock server for this time server: If not None, use this server instead of selecting one from serverGroup. (Used on manual assign) - excludeServersUUids: If not None, exclude this servers from selection. Used in case we check the availability of a server + excluded_servers_uuids: If not None, exclude this servers from selection. Used in case we check the availability of a server with some external method and we want to exclude it from selection because it has already failed. Returns: uuid of server assigned """ - if not userService.user: + if not userservice.user: raise exceptions.UDSException(_('No user assigned to service')) # Look for existing user asignation through properties - prop_name = self.property_name(userService.user) + prop_name = self.property_name(userservice.user) now = model_utils.sql_datetime() - excludeServersUUids = excludeServersUUids or set() + excluded_servers_uuids = excluded_servers_uuids or set() - with serverGroup.properties as props: + with server_group.properties as props: info: typing.Optional[types.servers.ServerCounter] = types.servers.ServerCounter.from_iterable( props.get(prop_name) ) # If server is forced, and server is part of the group, use it if server: if ( - server.groups.filter(uuid=serverGroup.uuid).exclude(uuid__in=excludeServersUUids).count() + server.groups.filter(uuid=server_group.uuid).exclude(uuid__in=excluded_servers_uuids).count() == 0 ): raise exceptions.UDSException(_('Server is not part of the group')) @@ -253,7 +253,7 @@ class ServerManager(metaclass=singleton.Singleton): # remove it from saved and use look for another one svr = models.Server.objects.filter(uuid=info.server_uuid).first() if not svr or ( - svr.maintenance_mode or svr.uuid in excludeServersUUids or svr.is_restrained() + svr.maintenance_mode or svr.uuid in excluded_servers_uuids or svr.is_restrained() ): info = None del props[prop_name] @@ -265,20 +265,20 @@ class ServerManager(metaclass=singleton.Singleton): try: with transaction.atomic(): best = self._find_best_server( - userService=userService, - serverGroup=serverGroup, + userservice=userservice, + server_group=server_group, now=now, - minMemoryMB=minMemoryMB, - excludeServersUUids=excludeServersUUids, + min_memory_mb=min_memory_mb, + excluded_servers_uuids=excluded_servers_uuids, ) info = types.servers.ServerCounter(best[0].uuid, 0) - best[0].locked_until = now + lockTime if lockTime else None + best[0].locked_until = now + lock_interval if lock_interval else None best[0].save(update_fields=['locked_until']) except exceptions.UDSException: # No more servers return None - elif lockTime: # If lockTime is set, update it - models.Server.objects.filter(uuid=info.server_uuid).update(locked_until=now + lockTime) + elif lock_interval: # If lockTime is set, update it + models.Server.objects.filter(uuid=info.server_uuid).update(locked_until=now + lock_interval) # Notify to server # Update counter @@ -293,7 +293,7 @@ class ServerManager(metaclass=singleton.Singleton): # Notify assgination in every case, even if reassignation to same server is made # This lets the server to keep track, if needed, of multi-assignations - self.notify_assign(bestServer, userService, serviceType, info.counter) + self.notify_assign(bestServer, userservice, service_type, info.counter) return info def release( diff --git a/server/src/uds/core/managers/user_service.py b/server/src/uds/core/managers/user_service.py index 1e69325f6..665565e82 100644 --- a/server/src/uds/core/managers/user_service.py +++ b/server/src/uds/core/managers/user_service.py @@ -62,8 +62,8 @@ if typing.TYPE_CHECKING: from uds import models logger = logging.getLogger(__name__) -traceLogger = logging.getLogger('traceLog') -operationsLogger = logging.getLogger('operationsLog') +trace_logger = logging.getLogger('traceLog') +operations_logger = logging.getLogger('operationsLog') class UserServiceManager(metaclass=singleton.Singleton): @@ -155,7 +155,7 @@ class UserServiceManager(metaclass=singleton.Singleton): in_use=False, ) - def _create_assigned_at_db_for_no_publication(self, service_pool: ServicePool, user: User) -> UserService: + def _create_assigned_user_service_at_db_from_pool(self, service_pool: ServicePool, user: User) -> UserService: """ __createCacheAtDb and __createAssignedAtDb uses a publication for create the UserService. There is cases where deployed services do not have publications (do not need them), so we need this method to create @@ -179,7 +179,7 @@ class UserServiceManager(metaclass=singleton.Singleton): """ Creates a new cache for the deployed service publication at level indicated """ - operationsLogger.info( + operations_logger.info( 'Creating a new cache element at level %s for publication %s', cacheLevel, publication, @@ -198,7 +198,7 @@ class UserServiceManager(metaclass=singleton.Singleton): # First, honor concurrent_creation_limit if self.can_grow_service_pool(service_pool) is False: # Cannot create new - operationsLogger.info( + operations_logger.info( 'Too many preparing services. Creation of assigned service denied by max preparing services parameter. (login storm with insufficient cache?).' ) raise ServiceNotReadyError() @@ -207,7 +207,7 @@ class UserServiceManager(metaclass=singleton.Singleton): publication = service_pool.active_publication() if publication: assigned = self._create_assigned_user_service_at_db(publication, user) - operationsLogger.info( + operations_logger.info( 'Creating a new assigned element for user %s for publication %s on pool %s', user.pretty_name, publication.revision, @@ -218,12 +218,12 @@ class UserServiceManager(metaclass=singleton.Singleton): f'Invalid publication creating service assignation: {service_pool.name} {user.pretty_name}' ) else: - operationsLogger.info( + operations_logger.info( 'Creating a new assigned element for user %s on pool %s', user.pretty_name, service_pool.name, ) - assigned = self._create_assigned_at_db_for_no_publication(service_pool, user) + assigned = self._create_assigned_user_service_at_db_from_pool(service_pool, user) assignedInstance = assigned.get_instance() state = assignedInstance.deploy_for_user(user) @@ -232,7 +232,7 @@ class UserServiceManager(metaclass=singleton.Singleton): return assigned - def create_from_assignable(self, service_pool: ServicePool, user: User, assignableId: str) -> UserService: + def create_from_assignable(self, service_pool: ServicePool, user: User, assignable_id: str) -> UserService: """ Creates an assigned service from an "assignable" id """ @@ -244,9 +244,9 @@ class UserServiceManager(metaclass=singleton.Singleton): publication = service_pool.active_publication() if publication: assigned = self._create_assigned_user_service_at_db(publication, user) - operationsLogger.info( + operations_logger.info( 'Creating an assigned element from assignable %s for user %s for publication %s on pool %s', - assignableId, + assignable_id, user.pretty_name, publication.revision, service_pool.name, @@ -256,20 +256,20 @@ class UserServiceManager(metaclass=singleton.Singleton): f'Invalid publication creating service assignation: {service_pool.name} {user.pretty_name}' ) else: - operationsLogger.info( + operations_logger.info( 'Creating an assigned element from assignable %s for user %s on pool %s', - assignableId, + assignable_id, user.pretty_name, service_pool.name, ) - assigned = self._create_assigned_at_db_for_no_publication(service_pool, user) + assigned = self._create_assigned_user_service_at_db_from_pool(service_pool, user) # Now, get from serviceInstance the data - assignedInstance = assigned.get_instance() - state = serviceInstance.assign_from_assignables(assignableId, user, assignedInstance) + assigned_userservice_instance = assigned.get_instance() + state = serviceInstance.assign_from_assignables(assignable_id, user, assigned_userservice_instance) # assigned.u(assignedInstance) - UserServiceOpChecker.make_unique(assigned, assignedInstance, state) + UserServiceOpChecker.make_unique(assigned, assigned_userservice_instance, state) return assigned @@ -307,7 +307,7 @@ class UserServiceManager(metaclass=singleton.Singleton): logger.debug('Cancel requested for a non running operation, performing removal instead') return self.remove(user_service) - operationsLogger.info('Canceling userService %s', user_service.name) + operations_logger.info('Canceling userService %s', user_service.name) user_service_instance = user_service.get_instance() if ( @@ -332,7 +332,7 @@ class UserServiceManager(metaclass=singleton.Singleton): """ with transaction.atomic(): userservice = UserService.objects.select_for_update().get(id=userservice.id) - operationsLogger.info('Removing userService %a', userservice.name) + operations_logger.info('Removing userService %a', userservice.name) if userservice.is_usable() is False and State.from_str(userservice.state).is_removable() is False: raise OperationException(_('Can\'t remove a non active element')) userservice.set_state(State.REMOVING) @@ -590,7 +590,7 @@ class UserServiceManager(metaclass=singleton.Singleton): if not user_service.deployed_service.service.get_type().can_reset: return - operationsLogger.info('Reseting %s', user_service) + operations_logger.info('Reseting %s', user_service) userServiceInstance = user_service.get_instance() try: @@ -821,7 +821,7 @@ class UserServiceManager(metaclass=singleton.Singleton): user_service, transportInstance.get_connection_info(user_service, user, ''), ) - traceLogger.info( + trace_logger.info( 'READY on service "%s" for user "%s" with transport "%s" (ip:%s)', user_service.name, userName, @@ -853,7 +853,7 @@ class UserServiceManager(metaclass=singleton.Singleton): log.LogSource.WEB, ) - traceLogger.error( + trace_logger.error( 'ERROR %s on service "%s" for user "%s" with transport "%s" (ip:%s)', serviceNotReadyCode, user_service.name, diff --git a/server/src/uds/core/services/service.py b/server/src/uds/core/services/service.py index ea13d8565..b772cf53c 100644 --- a/server/src/uds/core/services/service.py +++ b/server/src/uds/core/services/service.py @@ -339,13 +339,13 @@ class Service(Module): return [] def assign_from_assignables( - self, assignable_id: str, user: 'models.User', userservice: 'UserService' + self, assignable_id: str, user: 'models.User', userservice_instance: 'UserService' ) -> str: """ Assigns from it internal assignable list to an user args: - assignableId: Id of the assignable element + assignable_id: Id of the assignable element user: User to assign to userDeployment: User deployment to assign @@ -353,7 +353,11 @@ class Service(Module): Base implementation does nothing, to be overriden if needed Returns: - str: The state of the service after the assignation + str: The state of the user service after the assignation + + Note: + The state is the state of the "user service" after the assignation, not the state of the service itself. + This allows to process the assignation as an user service regular task, so it can be processed by the core. """ return State.FINISHED diff --git a/server/src/uds/core/services/specializations/fixed_machine/fixed_service.py b/server/src/uds/core/services/specializations/fixed_machine/fixed_service.py index 7486927e1..9de2cf1b1 100644 --- a/server/src/uds/core/services/specializations/fixed_machine/fixed_service.py +++ b/server/src/uds/core/services/specializations/fixed_machine/fixed_service.py @@ -95,6 +95,20 @@ class FixedService(services.Service, abc.ABC): # pylint: disable=too-many-publi tab=_('Machines'), old_field_name='useSnapshots', ) + + # This one replaces use_snapshots, and is used to select the snapshot type (No snapshot, recover snapshot and stop machine, recover snapshot and start machine) + snapshot_type = gui.ChoiceField( + label=_('Snapshot type'), + order=22, + default='0', + tooltip=_('If active, UDS will try to create an snapshot (if one already does not exists) before accessing a machine, and restore it after usage.'), + tab=_('Machines'), + choices=[ + gui.choice_item('no', _('No snapshot')), + gui.choice_item('stop', _('Recover snapshot and stop machine')), + gui.choice_item('start', _('Recover snapshot and start machine')), + ], + ) # Keep name as "machine" so we can use VCHelpers.getMachines machines = gui.MultiChoiceField( @@ -132,44 +146,56 @@ class FixedService(services.Service, abc.ABC): # pylint: disable=too-many-publi returns: str: the state to be processes by user service """ - return types.states.State.RUNNING + return types.states.State.FINISHED @abc.abstractmethod def get_machine_name(self, vmid: str) -> str: """ Returns the machine name for the given vmid """ - pass + raise NotImplementedError() @abc.abstractmethod def get_and_assign_machine(self) -> str: """ Gets automatically an assigns a machine + Returns the id of the assigned machine """ - pass + raise NotImplementedError() @abc.abstractmethod - def remove_and_free_machine(self, vmid: str) -> None: + def remove_and_free_machine(self, vmid: str) -> str: """ Removes and frees a machine + Returns an state (State.FINISHED is nothing to do left) """ - pass + raise NotImplementedError() @abc.abstractmethod def get_first_network_mac(self, vmid: str) -> str: - """If no mac, return empty string""" - pass + """If no mac, return empty string + Returns the first network mac of the machine + """ + raise NotImplementedError() @abc.abstractmethod def get_guest_ip_address(self, vmid: str) -> str: - pass + """Returns the guest ip address of the machine + """ + raise NotImplementedError() @abc.abstractmethod def enumerate_assignables(self) -> list[tuple[str, str]]: - pass + """ + Returns a list of tuples with the id and the name of the assignables + """ + raise NotImplementedError() @abc.abstractmethod def assign_from_assignables( - self, assignable_id: str, user: 'models.User', user_deployment: 'services.UserService' + self, assignable_id: str, user: 'models.User', userservice_instance: 'services.UserService' ) -> str: - pass + """ + Assigns a machine from the assignables + """ + raise NotImplementedError() diff --git a/server/src/uds/core/services/specializations/fixed_machine/fixed_userservice.py b/server/src/uds/core/services/specializations/fixed_machine/fixed_userservice.py index d9a3df661..e0c34fb6f 100644 --- a/server/src/uds/core/services/specializations/fixed_machine/fixed_userservice.py +++ b/server/src/uds/core/services/specializations/fixed_machine/fixed_userservice.py @@ -36,6 +36,7 @@ import enum import logging import typing import collections.abc +from webbrowser import Opera from uds.core import services, consts from uds.core.managers.user_service import UserServiceManager @@ -60,6 +61,9 @@ class Operation(enum.IntEnum): ERROR = 5 FINISH = 6 RETRY = 7 + SNAPSHOT_CREATE = 8 # to recall process_snapshot + SNAPSHOT_RECOVER = 9 # to recall process_snapshot + PROCESS_TOKEN = 10 UNKNOWN = 99 @@ -84,14 +88,27 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, _vmid = autoserializable.StringField(default='') _reason = autoserializable.StringField(default='') _task = autoserializable.StringField(default='') + _exec_state = autoserializable.StringField(default=State.RUNNING) _queue = autoserializable.ListField[Operation]() # Default is empty list - _create_queue: typing.ClassVar[typing.List[Operation]] = [ + _create_queue: typing.ClassVar[list[Operation]] = [ Operation.CREATE, + Operation.SNAPSHOT_CREATE, + Operation.PROCESS_TOKEN, Operation.START, Operation.FINISH, ] - _destrpy_queue: typing.ClassVar[typing.List[Operation]] = [Operation.REMOVE, Operation.FINISH] + _destroy_queue: typing.ClassVar[list[Operation]] = [ + Operation.REMOVE, + Operation.SNAPSHOT_RECOVER, + Operation.FINISH, + ] + _assign_queue: typing.ClassVar[list[Operation]] = [ + Operation.CREATE, + Operation.SNAPSHOT_CREATE, + Operation.PROCESS_TOKEN, + Operation.FINISH, + ] @typing.final def _get_current_op(self) -> Operation: @@ -175,17 +192,16 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, Deploys an service instance for an user. """ logger.debug('Deploying for user') - self._init_queue_for_deploy(False) + self._vmid = self.service().get_and_assign_machine() + self._queue = FixedUserService._create_queue.copy() # copy is needed to avoid modifying class var return self._execute_queue() @typing.final def assign(self, vmid: str) -> str: logger.debug('Assigning from VM {}'.format(vmid)) - return self._create(vmid) - - @typing.final - def _init_queue_for_deploy(self, for_level2: bool = False) -> None: - self._queue = FixedUserService._create_queue.copy() # copy is needed to avoid modifying class var + self._vmid = vmid + self._queue = FixedUserService._assign_queue.copy() # copy is needed to avoid modifying class var + return self._execute_queue() @typing.final def _execute_queue(self) -> str: @@ -198,17 +214,20 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, if op == Operation.FINISH: return State.FINISHED - fncs: collections.abc.Mapping[Operation, typing.Optional[collections.abc.Callable[[], str]]] = { + fncs: dict[Operation, collections.abc.Callable[[], None]] = { Operation.CREATE: self._create, Operation.RETRY: self._retry, Operation.START: self._start_machine, Operation.STOP: self._stop_machine, Operation.WAIT: self._wait, Operation.REMOVE: self._remove, + Operation.SNAPSHOT_CREATE: self._snapshot_create, + Operation.SNAPSHOT_RECOVER: self._snapshot_recover, + Operation.PROCESS_TOKEN: self._process_token, } try: - operation_runner: typing.Optional[collections.abc.Callable[[], str]] = fncs.get(op, None) + operation_runner: typing.Optional[collections.abc.Callable[[], None]] = fncs.get(op, None) if not operation_runner: return self._error(f'Unknown operation found at execution queue ({op})') @@ -221,7 +240,7 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, return self._error(str(e)) @typing.final - def _retry(self) -> str: + def _retry(self) -> None: """ Used to retry an operation In fact, this will not be never invoked, unless we push it twice, because @@ -229,62 +248,86 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, At executeQueue this return value will be ignored, and it will only be used at check_state """ - return State.FINISHED + pass @typing.final - def _wait(self) -> str: + def _wait(self) -> None: """ Executes opWait, it simply waits something "external" to end """ - return State.RUNNING + pass @typing.final - def _create(self, vmid: str = '') -> str: + def _create(self) -> None: """ Deploys a machine from template for user/cache """ - self._vmid = vmid or self.service().get_and_assign_machine() self._mac = self.service().get_first_network_mac(self._vmid) or '' self._name = self.service().get_machine_name(self._vmid) or f'VM-{self._vmid}' + @typing.final + def _snapshot_create(self) -> None: + """ + Creates a snapshot if needed + """ # Try to process snaptshots if needed - state = self.service().process_snapshot(remove=False, userservice_instace=self) + self._exec_state = self.service().process_snapshot(remove=False, userservice_instace=self) - if state == State.ERROR: - return state + @typing.final + def _snapshot_recover(self) -> None: + """ + Recovers a snapshot if needed + """ + self._exec_state = self.service().process_snapshot(remove=True, userservice_instace=self) + @typing.final + def _process_token(self) -> None: # If not to be managed by a token, "autologin" user if not self.service().get_token(): userService = self.db_obj() if userService: userService.set_in_use(True) - return state + self._exec_state = State.FINISHED - @typing.final - def _remove(self) -> str: + def _remove(self) -> None: """ Removes the snapshot if needed and releases the machine again """ - self.service().remove_and_free_machine(self._vmid) - - state = self.service().process_snapshot(remove=True, userservice_instace=self) - - return state - - @abc.abstractmethod - def _start_machine(self) -> str: - pass - - @abc.abstractmethod - def _stop_machine(self) -> str: - pass + self._exec_state = self.service().remove_and_free_machine(self._vmid) # Check methods def _create_checker(self) -> str: """ Checks the state of a deploy for an user or cache """ + return self._state_checker() + + def _snapshot_create_checker(self) -> str: + """ + Checks the state of a snapshot creation + """ + return self._state_checker() + + def _snapshot_recover_checker(self) -> str: + """ + Checks the state of a snapshot recovery + """ + return self._state_checker() + + def _process_token_checker(self) -> str: + """ + Checks the state of a token processing + """ + return self._state_checker() + + def _state_checker(self) -> str: + return self._exec_state + + def _retry_checker(self) -> str: + return State.FINISHED + + def _wait_checker(self) -> str: return State.FINISHED @abc.abstractmethod @@ -292,21 +335,28 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, """ Checks if machine has started """ - pass + raise NotImplementedError() + + @abc.abstractmethod + def _start_machine(self) -> None: + raise NotImplementedError() + + @abc.abstractmethod + def _stop_machine(self) -> None: + raise NotImplementedError() @abc.abstractmethod def _stop_checker(self) -> str: """ Checks if machine has stoped """ - pass + raise NotImplementedError() - @abc.abstractmethod def _removed_checker(self) -> str: """ Checks if a machine has been removed """ - pass + return self._exec_state @typing.final def check_state(self) -> str: @@ -322,13 +372,16 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, if op == Operation.FINISH: return State.FINISHED - fncs: collections.abc.Mapping[Operation, typing.Optional[collections.abc.Callable[[], str]]] = { + fncs: dict[Operation, collections.abc.Callable[[], str]] = { Operation.CREATE: self._create_checker, - Operation.RETRY: self._retry, - Operation.WAIT: self._wait, + Operation.RETRY: self._retry_checker, + Operation.WAIT: self._wait_checker, Operation.START: self._start_checker, Operation.STOP: self._stop_checker, Operation.REMOVE: self._removed_checker, + Operation.SNAPSHOT_CREATE: self._snapshot_create_checker, + Operation.SNAPSHOT_RECOVER: self._snapshot_recover_checker, + Operation.PROCESS_TOKEN: self._process_token_checker, } try: @@ -371,7 +424,7 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, """ Invoked for destroying a deployed service """ - self._queue = FixedUserService._destrpy_queue.copy() # copy is needed to avoid modifying class var + self._queue = FixedUserService._destroy_queue.copy() # copy is needed to avoid modifying class var return self._execute_queue() @typing.final @@ -385,7 +438,7 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, When administrator requests it, the cancel is "delayed" and not invoked directly. """ - logger.debug('Canceling %s with taskId=%s, vmId=%s', self._name, self._task, self._vmid) + logger.debug('Canceling %s with taskid=%s, vmid=%s', self._name, self._task, self._vmid) return self.destroy() @staticmethod @@ -399,6 +452,9 @@ class FixedUserService(services.UserService, autoserializable.AutoSerializable, Operation.ERROR: 'error', Operation.FINISH: 'finish', Operation.RETRY: 'retry', + Operation.SNAPSHOT_CREATE: 'snapshot_create', + Operation.SNAPSHOT_RECOVER: 'snapshot_recover', + Operation.PROCESS_TOKEN: 'process_token', }.get(op, '????') def _debug(self, txt: str) -> None: diff --git a/server/src/uds/services/Proxmox/deployment_fixed.py b/server/src/uds/services/Proxmox/deployment_fixed.py index 77e2a38a9..8723c7738 100644 --- a/server/src/uds/services/Proxmox/deployment_fixed.py +++ b/server/src/uds/services/Proxmox/deployment_fixed.py @@ -65,6 +65,7 @@ class ProxmoxFixedUserService(FixedUserService, autoserializable.AutoSerializabl # : Recheck every ten seconds by default (for task methods) suggested_delay = 4 + def _store_task(self, upid: 'client.types.UPID') -> None: self._task = '\t'.join([upid.node, upid.upid]) @@ -110,20 +111,18 @@ class ProxmoxFixedUserService(FixedUserService, autoserializable.AutoSerializabl def error(self, reason: str) -> str: return self._error(reason) - def _start_machine(self) -> str: + def _start_machine(self) -> None: try: vminfo = self.service().get_machine_info(int(self._vmid)) except client.ProxmoxConnectionError: - return self._retry_later() + self._retry_later() except Exception as e: raise Exception('Machine not found on start machine') from e if vminfo.status == 'stopped': self._store_task(self.service().start_machine(int(self._vmid))) - return State.RUNNING - - def _stop_machine(self) -> str: + def _stop_machine(self) -> None: try: vm_info = self.service().get_machine_info(int(self._vmid)) except Exception as e: @@ -133,8 +132,6 @@ class ProxmoxFixedUserService(FixedUserService, autoserializable.AutoSerializabl logger.debug('Stopping machine %s', vm_info) self._store_task(self.service().stop_machine(int(self._vmid))) - return State.RUNNING - # Check methods def _check_task_finished(self) -> str: if self._task == '': @@ -156,12 +153,6 @@ class ProxmoxFixedUserService(FixedUserService, autoserializable.AutoSerializabl return State.RUNNING # Check methods - def _create_checker(self) -> str: - """ - Checks the state of a deploy for an user or cache - """ - return State.FINISHED - def _start_checker(self) -> str: """ Checks if machine has started @@ -173,9 +164,3 @@ class ProxmoxFixedUserService(FixedUserService, autoserializable.AutoSerializabl Checks if machine has stoped """ return self._check_task_finished() - - def _removed_checker(self) -> str: - """ - Checks if a machine has been removed - """ - return self._check_task_finished() diff --git a/server/src/uds/services/Proxmox/service_fixed.py b/server/src/uds/services/Proxmox/service_fixed.py index 2e65e8718..80084e8c8 100644 --- a/server/src/uds/services/Proxmox/service_fixed.py +++ b/server/src/uds/services/Proxmox/service_fixed.py @@ -201,7 +201,7 @@ class ProxmoxFixedService(FixedService): # pylint: disable=too-many-public-meth except Exception as e: self.do_log(log.LogLevel.WARNING, 'Could not create SNAPSHOT for this VM. ({})'.format(e)) - return types.states.State.RUNNING + return types.states.State.FINISHED def get_and_assign_machine(self) -> str: found_vmid: typing.Optional[int] = None @@ -245,8 +245,10 @@ class ProxmoxFixedService(FixedService): # pylint: disable=too-many-public-meth def get_machine_name(self, vmid: str) -> str: return self.parent().get_machine_info(int(vmid)).name or '' - def remove_and_free_machine(self, vmid: str) -> None: + def remove_and_free_machine(self, vmid: str) -> str: try: self._save_assigned_machines(self._get_assigned_machines() - {str(vmid)}) # Remove from assigned + return types.states.State.FINISHED except Exception as e: logger.warn('Cound not save assigned machines on fixed pool: %s', e) + raise diff --git a/server/tests/core/managers/test_servers_managed.py b/server/tests/core/managers/test_servers_managed.py index aeb340e52..4b80d4a89 100644 --- a/server/tests/core/managers/test_servers_managed.py +++ b/server/tests/core/managers/test_servers_managed.py @@ -84,9 +84,9 @@ class ServerManagerManagedServersTest(UDSTestCase): # commodity call to assign self.assign = functools.partial( self.manager.assign, - serverGroup=self.registered_servers_group, - serviceType=types.services.ServiceType.VDI, - minMemoryMB=MIN_TEST_MEMORY_MB, + server_group=self.registered_servers_group, + service_type=types.services.ServiceType.VDI, + min_memory_mb=MIN_TEST_MEMORY_MB, ) self.all_uuids: list[str] = list( self.registered_servers_group.servers.all().values_list('uuid', flat=True) @@ -217,7 +217,7 @@ class ServerManagerManagedServersTest(UDSTestCase): with self.create_mock_api_requester() as mockServerApiRequester: # Assign all user services with lock for userService in self.user_services[:NUM_REGISTEREDSERVERS]: - assignation = self.assign(userService, lockTime=datetime.timedelta(seconds=1)) + assignation = self.assign(userService, lock_interval=datetime.timedelta(seconds=1)) if assignation is None: self.fail('Assignation returned None') return # For mypy @@ -231,12 +231,12 @@ class ServerManagerManagedServersTest(UDSTestCase): # Next one should fail returning None self.assertIsNone( - self.assign(self.user_services[NUM_REGISTEREDSERVERS], lockTime=datetime.timedelta(seconds=1)) + self.assign(self.user_services[NUM_REGISTEREDSERVERS], lock_interval=datetime.timedelta(seconds=1)) ) # Wait a second, and try again, it should work time.sleep(1) - self.assign(self.user_services[NUM_REGISTEREDSERVERS], lockTime=datetime.timedelta(seconds=1)) + self.assign(self.user_services[NUM_REGISTEREDSERVERS], lock_interval=datetime.timedelta(seconds=1)) # notify_release should has been called once self.assertEqual(mockServerApiRequester.return_value.notify_release.call_count, 1) @@ -247,7 +247,7 @@ class ServerManagerManagedServersTest(UDSTestCase): for assignations in range(2): # Second pass will get current assignation, not new ones for elementNumber, userService in enumerate(self.user_services[:NUM_REGISTEREDSERVERS]): # Ensure locking server, so we have to use every server only once - assignation = self.assign(userService, lockTime=datetime.timedelta(seconds=32)) + assignation = self.assign(userService, lock_interval=datetime.timedelta(seconds=32)) self.assertEqual( serverApiRequester.notify_assign.call_count, assignations * NUM_REGISTEREDSERVERS + elementNumber + 1, @@ -265,7 +265,7 @@ class ServerManagerManagedServersTest(UDSTestCase): # Trying to lock a new one, should fail self.assertIsNone( - self.assign(self.user_services[NUM_REGISTEREDSERVERS], lockTime=datetime.timedelta(seconds=32)) + self.assign(self.user_services[NUM_REGISTEREDSERVERS], lock_interval=datetime.timedelta(seconds=32)) ) # All servers should be locked diff --git a/server/tests/core/managers/test_servers_unmanaged.py b/server/tests/core/managers/test_servers_unmanaged.py index be7c146b0..194e58a30 100644 --- a/server/tests/core/managers/test_servers_unmanaged.py +++ b/server/tests/core/managers/test_servers_unmanaged.py @@ -78,8 +78,8 @@ class ServerManagerUnmanagedServersTest(UDSTestCase): # commodity call to assign self.assign = functools.partial( self.manager.assign, - serverGroup=self.registered_servers_group, - serviceType=types.services.ServiceType.VDI, + server_group=self.registered_servers_group, + service_type=types.services.ServiceType.VDI, ) self.all_uuids: list[str] = list( self.registered_servers_group.servers.all().values_list('uuid', flat=True) @@ -176,7 +176,7 @@ class ServerManagerUnmanagedServersTest(UDSTestCase): with self.createMockApiRequester() as mockServerApiRequester: # Assign all user services with lock for userService in self.user_services[:NUM_REGISTEREDSERVERS]: - assignation = self.assign(userService, lockTime=datetime.timedelta(seconds=1.1)) + assignation = self.assign(userService, lock_interval=datetime.timedelta(seconds=1.1)) if assignation is None: self.fail('Assignation returned None') return # For mypy @@ -191,12 +191,12 @@ class ServerManagerUnmanagedServersTest(UDSTestCase): # Next one should fail with aa None return self.assertIsNone( - self.assign(self.user_services[NUM_REGISTEREDSERVERS], lockTime=datetime.timedelta(seconds=1)) + self.assign(self.user_services[NUM_REGISTEREDSERVERS], lock_interval=datetime.timedelta(seconds=1)) ) # Wait a bit more than a second, and try again, it should work time.sleep(1.1) - self.assign(self.user_services[NUM_REGISTEREDSERVERS], lockTime=datetime.timedelta(seconds=1)) + self.assign(self.user_services[NUM_REGISTEREDSERVERS], lock_interval=datetime.timedelta(seconds=1)) # notify_release should has been called once self.assertEqual(mockServerApiRequester.return_value.notify_release.call_count, 1) diff --git a/server/tests/core/services/__init__.py b/server/tests/core/services/__init__.py new file mode 100644 index 000000000..2e5c732bd --- /dev/null +++ b/server/tests/core/services/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" +Author: Adolfo Gómez, dkmaster at dkmon dot com +""" diff --git a/server/tests/core/services/specializations/__init__.py b/server/tests/core/services/specializations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/server/tests/core/services/specializations/test_fixedservice.py b/server/tests/core/services/specializations/test_fixedservice.py new file mode 100644 index 000000000..ff8ba3f28 --- /dev/null +++ b/server/tests/core/services/specializations/test_fixedservice.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +# +# Copyright (c) 2022 Virtual Cable S.L.U. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" +@author: Adolfo Gómez, dkmaster at dkmon dot com +""" +import typing +from uds import models +from uds.core import types, services +from ....utils.test import UDSTestCase + +from uds.core.services.specializations.fixed_machine import fixed_service, fixed_userservice + + +class FixedServiceTest(UDSTestCase): + pass + + +class FixedUserService(fixed_userservice.FixedUserService): + started: bool = False + counter: int = 0 + + def _start_machine(self) -> None: + self.started = True + self.counter = 2 + + def _stop_machine(self) -> None: + self.started = False + self.counter = 2 + + def _start_checker(self) -> str: + self.counter = self.counter - 1 + if self.counter <= 0: + return types.states.State.FINISHED + return types.states.State.RUNNING + + def _stop_checker(self) -> str: + self.counter = self.counter - 1 + if self.counter <= 0: + return types.states.State.FINISHED + return types.states.State.RUNNING + + +class FixedService(fixed_service.FixedService): + token = fixed_service.FixedService.token + snapshot_type = fixed_service.FixedService.snapshot_type + machines = fixed_service.FixedService.machines + + snapshot_proccessed: bool = False + is_remove_snapshot: bool = False + assigned_machine: str = '' + + user_service_type = FixedUserService + + def process_snapshot(self, remove: bool, userservice_instace: fixed_userservice.FixedUserService) -> str: + self.snapshot_proccessed = True + self.is_remove_snapshot = remove + return super().process_snapshot(remove, userservice_instace) + + def get_machine_name(self, vmid: str) -> str: + return f'Machine {vmid}' + + def get_and_assign_machine(self) -> str: + self.assigned_machine = 'assigned' + return self.assigned_machine + + def remove_and_free_machine(self, vmid: str) -> str: + self.assigned_machine = '' + return types.states.State.FINISHED + + def get_first_network_mac(self, vmid: str) -> str: + raise NotImplementedError() + + def get_guest_ip_address(self, vmid: str) -> str: + raise NotImplementedError() + + def enumerate_assignables(self) -> list[tuple[str, str]]: + """ + Returns a list of tuples with the id and the name of the assignables + """ + return [ + ('1', 'Machine 1'), + ('2', 'Machine 2'), + ('3', 'Machine 3'), + ] + + def assign_from_assignables( + self, assignable_id: str, user: 'models.User', userservice_instance: 'services.UserService' + ) -> str: + """ + Assigns a machine from the assignables + """ + return types.states.State.FINISHED + + +class FixedProvider(services.provider.ServiceProvider): + offers = [FixedService]