From 5f74c7fca890c2864242069b23db3d6d8b843fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20G=C3=B3mez=20Garc=C3=ADa?= Date: Fri, 2 Mar 2018 00:51:16 +0100 Subject: [PATCH] * Changed the "persitent" management on case of new publication. Now, if a machine is considered "persistent", on a new publication, UDS Will not remove assigned services, nor will remove services of old publications on logout. --- .../uds/core/managers/PublicationManager.py | 13 ++- .../uds/core/managers/UserServiceManager.py | 19 ++-- .../src/uds/core/osmanagers/BaseOsManager.py | 11 ++- .../LinuxOsManager/LinuxOsManager.py | 3 + .../WindowsOsManager/WindowsOsManager.py | 3 + .../OpenStack/openStack/UDSOpenStackClient.py | 91 ++----------------- 6 files changed, 47 insertions(+), 93 deletions(-) diff --git a/server/src/uds/core/managers/PublicationManager.py b/server/src/uds/core/managers/PublicationManager.py index 1423f455..78bd0661 100644 --- a/server/src/uds/core/managers/PublicationManager.py +++ b/server/src/uds/core/managers/PublicationManager.py @@ -53,6 +53,7 @@ class PublicationOldMachinesCleaner(DelayedTask): ''' This delayed task is for removing a pending "removable" publication ''' + def __init__(self, publicationId): super(PublicationOldMachinesCleaner, self).__init__() self._id = publicationId @@ -77,6 +78,7 @@ class PublicationLauncher(DelayedTask): ''' This delayed task if for launching a new publication ''' + def __init__(self, publish): super(PublicationLauncher, self).__init__() self._publishId = publish.id @@ -108,6 +110,7 @@ class PublicationFinishChecker(DelayedTask): ''' This delayed task is responsible of checking if a publication is finished ''' + def __init__(self, servicePoolPub): super(PublicationFinishChecker, self).__init__() self._publishId = servicePoolPub.id @@ -128,8 +131,14 @@ class PublicationFinishChecker(DelayedTask): for old in servicePoolPub.deployed_service.publications.filter(state=State.USABLE): old.state = State.REMOVABLE old.save() - pc = PublicationOldMachinesCleaner(old.id) - pc.register(GlobalConfig.SESSION_EXPIRE_TIME.getInt(True) * 3600, 'pclean-' + str(old.id), True) + + osm = servicePoolPub.deployed_service.osmanager + # If os manager says "machine is persistent", do not tray to delete "previous version" assigned machines + doPublicationCleanup = True if osm is None else not osm.getInstance().isPersistent() + + if doPublicationCleanup: + pc = PublicationOldMachinesCleaner(old.id) + pc.register(GlobalConfig.SESSION_EXPIRE_TIME.getInt(True) * 3600, 'pclean-' + str(old.id), True) servicePoolPub.setState(State.USABLE) servicePoolPub.deployed_service.markOldUserServicesAsRemovables(servicePoolPub) diff --git a/server/src/uds/core/managers/UserServiceManager.py b/server/src/uds/core/managers/UserServiceManager.py index d80ec2dd..654f6fdb 100644 --- a/server/src/uds/core/managers/UserServiceManager.py +++ b/server/src/uds/core/managers/UserServiceManager.py @@ -51,7 +51,7 @@ import requests import json import logging -__updated__ = '2018-02-16' +__updated__ = '2018-03-02' logger = logging.getLogger(__name__) traceLogger = logging.getLogger('traceLog') @@ -455,12 +455,17 @@ class UserServiceManager(object): This method is used by UserService when a request for setInUse(False) is made This checks that the service can continue existing or not ''' - # uService = UserService.objects.get(id=uService.id) - if uService.publication is None: - return - if uService.publication.id != uService.deployed_service.activePublication().id: - logger.debug('Old revision of user service, marking as removable: {0}'.format(uService)) - uService.remove() + osm = uService.deployed_service.osmanager + # If os manager says "machine is persistent", do not tray to delete "previous version" assigned machines + doPublicationCleanup = True if osm is None else not osm.getInstance().isPersistent() + + if doPublicationCleanup: + if uService.publication is None: + return + + if uService.publication.id != uService.deployed_service.activePublication().id: + logger.debug('Old revision of user service, marking as removable: {0}'.format(uService)) + uService.remove() def notifyReadyFromOsManager(self, uService, data): try: diff --git a/server/src/uds/core/osmanagers/BaseOsManager.py b/server/src/uds/core/osmanagers/BaseOsManager.py index a1dc0aa4..61e1db48 100644 --- a/server/src/uds/core/osmanagers/BaseOsManager.py +++ b/server/src/uds/core/osmanagers/BaseOsManager.py @@ -42,7 +42,7 @@ from uds.core import Module import six -__updated__ = '2017-10-02' +__updated__ = '2018-03-02' STORAGE_KEY = 'osmk' @@ -177,11 +177,9 @@ class OSManager(Module): def logKnownIp(self, userService, ip): userService.logIP(ip) - def toReady(self, userService): userService.setProperty('loginsCounter', '0') - def loggedIn(self, userService, userName=None, save=True): ''' This method: @@ -262,6 +260,13 @@ class OSManager(Module): if save: userService.save() + def isPersistent(self): + ''' + When a publication if finished, old assigned machines will be removed if this value is True. + Defaults to False + ''' + return False + def __str__(self): return "Base OS Manager" diff --git a/server/src/uds/osmanagers/LinuxOsManager/LinuxOsManager.py b/server/src/uds/osmanagers/LinuxOsManager/LinuxOsManager.py index 3e5df5d1..28c72fe5 100644 --- a/server/src/uds/osmanagers/LinuxOsManager/LinuxOsManager.py +++ b/server/src/uds/osmanagers/LinuxOsManager/LinuxOsManager.py @@ -197,6 +197,9 @@ class LinuxOsManager(osmanagers.OSManager): if self._onLogout == 'remove': userService.remove() + def isPersistent(self): + return not self._onLogout == 'remove' + def checkState(self, service): logger.debug('Checking state for service {0}'.format(service)) return State.RUNNING diff --git a/server/src/uds/osmanagers/WindowsOsManager/WindowsOsManager.py b/server/src/uds/osmanagers/WindowsOsManager/WindowsOsManager.py index d17310aa..fe1f33e6 100644 --- a/server/src/uds/osmanagers/WindowsOsManager/WindowsOsManager.py +++ b/server/src/uds/osmanagers/WindowsOsManager/WindowsOsManager.py @@ -215,6 +215,9 @@ class WindowsOsManager(osmanagers.OSManager): if self._onLogout == 'remove': userService.remove() + def isPersistent(self): + return not self._onLogout == 'remove' + def checkState(self, service): logger.debug('Checking state for service {0}'.format(service)) return State.RUNNING diff --git a/server/src/uds/services/OpenStack/openStack/UDSOpenStackClient.py b/server/src/uds/services/OpenStack/openStack/UDSOpenStackClient.py index c76d7b2d..6daeac7c 100644 --- a/server/src/uds/services/OpenStack/openStack/UDSOpenStackClient.py +++ b/server/src/uds/services/OpenStack/openStack/UDSOpenStackClient.py @@ -33,33 +33,26 @@ # pylint: disable=maybe-no-member,protected-access from django.utils.translation import ugettext as _ -from uds.core.util.Cache import Cache - import logging import requests import json import dateutil.parser -import hashlib import six - -__updated__ = '2017-11-13' +__updated__ = '2018-03-02' logger = logging.getLogger(__name__) # Required: Authentication v3 - -# This is a vary basic implementation for what we need from openstack +# This is an implementation for what we need from openstack # This does not includes (nor it is intention) full API implementation, just the parts we need -# Theese are related to auth, compute & network basically - -# In case we Cache time for endpoints. This is more likely to not change never, so we will tray to keep it as long as we can (1 hour for example?) -# ENDPOINTS_TIMEOUT = 1 * 3600 +# These are related to auth, compute & network basically # Do not verify SSL conections right now VERIFY_SSL = False + # Helpers def ensureResponseIsValid(response, errMsg=None): if response.ok is False: @@ -96,28 +89,30 @@ def getRecurringUrlJson(url, headers, key, params=None, errMsg=None, timeout=10) # Decorators def authRequired(func): + def ensurer(obj, *args, **kwargs): obj.ensureAuthenticated() try: return func(obj, *args, **kwargs) except Exception as e: logger.error('Got error {} for openstack'.format(e)) - obj._cleanCache() # On any request error, force next time auth raise + return ensurer + def authProjectRequired(func): + def ensurer(obj, *args, **kwargs): if obj._projectId is None: raise Exception('Need a project for method {}'.format(func)) obj.ensureAuthenticated() return func(obj, *args, **kwargs) + return ensurer class Client(object): - cache = Cache('uds-openstack') - PUBLIC = 'public' PRIVATE = 'private' INTERNAL = 'url' @@ -138,18 +133,6 @@ class Client(object): self._authUrl = 'http{}://{}:{}/'.format('s' if useSSL else '', host, port) - # Generates a hash for auth + credentials - h = hashlib.md5() - h.update(six.binary_type(host)) - h.update(six.binary_type(port)) - h.update(six.binary_type(domain)) - h.update(six.binary_type(username)) - h.update(six.binary_type(password)) - h.update(six.binary_type(useSSL)) - h.update(six.binary_type(projectId)) - h.update(six.binary_type(region)) - self._cacheKey = h.hexdigest() - def _getEndpointFor(self, type_): # If no region is indicatad, first endpoint is returned for i in self._catalog: if i['type'] == type_: @@ -164,38 +147,7 @@ class Client(object): return headers - def _getFromCache(self): - cached = self.cache.get(self._cacheKey) - if cached is not None: - self._authenticated = True - self._tokenId = cached['tokenId'] - # Extract the token id - self._userId = cached['userId'] - self._projectId = cached['projectId'] - self._catalog = cached['catalog'] - - return True - - return False - - def _saveToCache(self, validity=600): - self.cache.put(self._cacheKey, - { - 'tokenId': self._tokenId, - 'userId': self._userId, - 'projectId': self._projectId, - 'catalog': self._catalog - }, - validity - 60) # We substract some seconds to allow some time desynchronization - - def _clearCache(self): - self.cache.remove(self._cacheKey) - def authPassword(self): - # If cached data exists, use it as auth - if self._getFromCache() is True: - return - data = { 'auth': { 'identity': { @@ -239,20 +191,16 @@ class Client(object): self._userId = token['user']['id'] validity = (dateutil.parser.parse(token['expires_at']).replace(tzinfo=None) - dateutil.parser.parse(token['issued_at']).replace(tzinfo=None)).seconds - 60 - logger.debug('The token {} will be valid for {}'.format(self._tokenId, validity)) - # Now, if endpoints are present (only if tenant was specified), store & cache them + # Now, if endpoints are present (only if tenant was specified), store them if self._projectId is not None: self._catalog = token['catalog'] - self._saveToCache(validity) - def ensureAuthenticated(self): if self._authenticated is False: self.authPassword() - @authRequired def listProjects(self): return getRecurringUrlJson(self._authUrl + 'v3/users/{user_id}/projects'.format(user_id=self._userId), @@ -261,7 +209,6 @@ class Client(object): errMsg='List Projects', timeout=self._timeout) - @authRequired def listRegions(self): return getRecurringUrlJson(self._authUrl + 'v3/regions/', @@ -270,7 +217,6 @@ class Client(object): errMsg='List Regions', timeout=self._timeout) - @authProjectRequired def listServers(self, detail=False, params=None): path = '/servers/' + 'detail' if detail is True else '' @@ -281,7 +227,6 @@ class Client(object): errMsg='List Vms', timeout=self._timeout) - @authProjectRequired def listImages(self): return getRecurringUrlJson(self._getEndpointFor('image') + '/v2/images?status=active', @@ -290,7 +235,6 @@ class Client(object): errMsg='List Images', timeout=self._timeout) - @authProjectRequired def listVolumeTypes(self): return getRecurringUrlJson(self._getEndpointFor('volumev2') + '/types', @@ -299,7 +243,6 @@ class Client(object): errMsg='List Volume Types', timeout=self._timeout) - @authProjectRequired def listVolumes(self): # self._getEndpointFor('volumev2') + '/volumes' @@ -309,7 +252,6 @@ class Client(object): errMsg='List Volumes', timeout=self._timeout) - @authProjectRequired def listVolumeSnapshots(self, volumeId=None): for s in getRecurringUrlJson(self._getEndpointFor('volumev2') + '/snapshots', @@ -320,7 +262,6 @@ class Client(object): if volumeId is None or s['volume_id'] == volumeId: yield s - @authProjectRequired def listAvailabilityZones(self): for az in getRecurringUrlJson(self._getEndpointFor('compute') + '/os-availability-zone', @@ -331,7 +272,6 @@ class Client(object): if az['zoneState']['available'] is True: yield az['zoneName'] - @authProjectRequired def listFlavors(self): return getRecurringUrlJson(self._getEndpointFor('compute') + '/flavors', @@ -340,7 +280,6 @@ class Client(object): errMsg='List Flavors', timeout=self._timeout) - @authProjectRequired def listNetworks(self): return getRecurringUrlJson(self._getEndpointFor('network') + '/v2.0/networks', @@ -372,7 +311,6 @@ class Client(object): errMsg='List security groups', timeout=self._timeout) - @authProjectRequired def getServer(self, serverId): r = requests.get(self._getEndpointFor('compute') + '/servers/{server_id}'.format(server_id=serverId), @@ -396,7 +334,6 @@ class Client(object): return v - @authProjectRequired def getSnapshot(self, snapshotId): ''' @@ -414,7 +351,6 @@ class Client(object): return v - @authProjectRequired def updateSnapshot(self, snapshotId, name=None, description=None): data = { 'snapshot': {} } @@ -436,7 +372,6 @@ class Client(object): return v - @authProjectRequired def createVolumeSnapshot(self, volumeId, name, description=None): description = 'UDS Snapshot' if description is None else description @@ -461,7 +396,6 @@ class Client(object): return r.json()['snapshot'] - @authProjectRequired def createVolumeFromSnapshot(self, snapshotId, name, description=None): description = 'UDS Volume' if description is None else description @@ -520,7 +454,6 @@ class Client(object): return r.json()['server'] - @authProjectRequired def deleteServer(self, serverId): r = requests.post(self._getEndpointFor('compute') + '/servers/{server_id}/action'.format(server_id=serverId), @@ -533,7 +466,6 @@ class Client(object): # This does not returns anything - @authProjectRequired def deleteSnapshot(self, snapshotId): r = requests.delete(self._getEndpointFor('volumev2') + '/snapshots/{snapshot_id}'.format(snapshot_id=snapshotId), @@ -545,7 +477,6 @@ class Client(object): # Does not returns a message body - @authProjectRequired def startServer(self, serverId): r = requests.post(self._getEndpointFor('compute') + '/servers/{server_id}/action'.format(server_id=serverId), @@ -558,7 +489,6 @@ class Client(object): # This does not returns anything - @authProjectRequired def stopServer(self, serverId): r = requests.post(self._getEndpointFor('compute') + '/servers/{server_id}/action'.format(server_id=serverId), @@ -589,7 +519,6 @@ class Client(object): ensureResponseIsValid(r, 'Resuming server') - def testConnection(self): # First, ensure requested api is supported # We need api version 3.2 or greater