* Added optimization for serialization/deserialization (now it get cached with db object)

* Fixed model so now we invoke initGui before getting values on fillInstanceFields
* Upgraded Max creating & Max removing limits so now they can be tied to an specific provider instead of a global variable
* Tested & Fixed OpenStack. Now it seems to work as it should do.
* Several minor fixes/improvements

server/src/uds/REST/model.py

@@ -54,7 +54,7 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-__updated__ = '2016-02-12'
+__updated__ = '2016-03-09'
 
 
 # a few constants
@@ -222,7 +222,9 @@ class BaseModelHandler(Handler):
         :param res: Dictionary to "extend" with instance key-values pairs
         '''
         if hasattr(item, 'getInstance'):
-            for key, value in item.getInstance().valuesDict().iteritems():
+            i = item.getInstance()
+            i.initGui()  # Defaults & stuff
+            for key, value in i.valuesDict().iteritems():
                 if type(value) in (unicode, str):
                     value = {"true": True, "false": False}.get(value, value)  # Translate "true" & "false" to True & False (booleans)
                 logger.debug('{0} = {1}'.format(key, value))

server/src/uds/core/managers/UserServiceManager.py

@@ -51,7 +51,7 @@ import requests
 import json
 import logging
 
-__updated__ = '2016-02-18'
+__updated__ = '2016-03-09'
 
 logger = logging.getLogger(__name__)
 
@@ -322,9 +322,11 @@ class UserServiceManager(object):
     def canRemoveServiceFromDeployedService(self, ds):
         '''
         checks if we can do a "remove" from a deployed service
+        serviceIsntance is just a helper, so if we already have unserialized deployedService
         '''
         removing = self.getServicesInStateForProvider(ds.service.provider_id, State.REMOVING)
-        if removing >= GlobalConfig.MAX_REMOVING_SERVICES.getInt() and GlobalConfig.IGNORE_LIMITS.getBool() is False:
+        serviceInstance = ds.service.getInstance()
+        if removing >= serviceInstance.parent().getMaxRemovingServices() and serviceInstance.parent().getIgnoreLimits() is False:
             return False
         return True
 
@@ -333,7 +335,8 @@ class UserServiceManager(object):
         Checks if we can start a new service
         '''
         preparing = self.getServicesInStateForProvider(ds.service.provider_id, State.PREPARING)
-        if preparing >= GlobalConfig.MAX_PREPARING_SERVICES.getInt() and GlobalConfig.IGNORE_LIMITS.getBool() is False:
+        serviceInstance = ds.service.getInstance()
+        if preparing >= serviceInstance.parent().getMaxPreparingServices() and serviceInstance.parent().getIgnoreLimits() is False:
             return False
         return True
 

server/src/uds/core/services/BaseService.py

@@ -37,7 +37,7 @@ from uds.core import Module
 from uds.core.transports import protocols
 from . import types
 
-__updated__ = '2016-02-26'
+__updated__ = '2016-03-09'
 
 
 class Service(Module):

server/src/uds/core/services/BaseServiceProvider.py

@@ -33,11 +33,13 @@
 from __future__ import unicode_literals
 
 from uds.core import Module
+from uds.core.util.Config import GlobalConfig
+from uds.core.ui.UserInterface import gui
 import logging
 
 logger = logging.getLogger(__name__)
 
-__updated__ = '2014-03-22'
+__updated__ = '2016-03-09'
 
 
 class ServiceProvider(Module):
@@ -103,6 +105,22 @@ class ServiceProvider(Module):
     # : your own py:meth:`uds.core.BaseModule.BaseModule.icon` method.
     iconFile = 'provider.png'
 
+    # : This defines the maximum number of concurrent services that should be in state "in preparation" for this provider
+    # : Default is return the GlobalConfig value of GlobalConfig.MAX_PREPARING_SERVICES
+    # : Note: this variable can be either a fixed value (integer, string) or a Gui text field (with a .value)
+    maxPreparingServices = None
+
+    # : This defines the maximum number of concurrent services that should be in state "removing" for this provider
+    # : Default is return the GlobalConfig value of GlobalConfig.MAX_REMOVING_SERVICES
+    # : Note: this variable can be either a fixed value (integer, string) or a Gui text field (with a .value)
+    maxRemovingServices = None
+
+    # : This defines if the limits (max.. vars) should be taken into accout or simply ignored
+    # : Default is return the GlobalConfig value of GlobalConfig.IGNORE_LIMITS
+    # : Note: this variable can be either a fixed value (integer, string) or a Gui text field (with a .value)
+    ignoreLimits = None
+
+
     @classmethod
     def getServicesTypes(cls):
         '''
@@ -155,6 +173,28 @@ class ServiceProvider(Module):
         '''
         pass
 
+    def getMaxPreparingServices(self):
+        val = self.maxPreparingServices
+        if val is None:
+            val = self.maxPreparingServices = GlobalConfig.MAX_PREPARING_SERVICES.getInt(force=True)  # Recover global an cache till restart
+
+        return int(getattr(val, 'value', val))
+
+    def getMaxRemovingServices(self):
+        val = self.maxRemovingServices
+        if val is None:
+            val = self.maxRemovingServices = GlobalConfig.MAX_REMOVING_SERVICES.getInt(force=True)  # Recover global an cache till restart
+
+        return int(getattr(val, 'value', val))
+
+    def getIgnoreLimits(self):
+        val = self.ignoreLimits
+        if val is None:
+            val = self.ignoreLimits = GlobalConfig.IGNORE_LIMITS.getBool(force=True)  # Recover global an cache till restart
+
+        val = getattr(val, 'value', val)
+        return val is True or val == gui.TRUE
+
     def __str__(self):
         '''
         Basic implementation, mostly used for debuging and testing, never used

server/src/uds/core/ui/UserInterface.py

@@ -250,9 +250,12 @@ class gui(object):
         @property
         def value(self):
             '''
-            Obtains the stored value
+            Obtains the stored value.
+            If the stored value is None (this will only happens if value is forced to be so, by default empty value is ''),
+            returns default value instead.
+            This is mainly used for hidden fields, so we have correctly initialized
             '''
-            return self._data['value']
+            return self._data['value'] if self._data['value'] is not None else self.defValue
 
         @value.setter
         def value(self, value):
@@ -850,9 +853,13 @@ class UserInterface(object):
                 dic[k] = gui.convertToList(v.value)
             elif v.isType(gui.InputField.MULTI_CHOICE_TYPE):
                 dic[k] = gui.convertToChoices(v.value)
+#            elif v.isType(gui.InputField.HIDDEN_TYPE) and v.isSerializable() is False and v.value == '':
+#                # This solves initializing some vars used to "pass" values to callbacks of form editors.
+#                # Now, we only need to initialize the hidden form fields on initGue
+#                dic[k] = v.defValue
             else:
                 dic[k] = v.value
-        logger.debug('Dict: {0}'.format(dic))
+        logger.debug('Values Dict: {0}'.format(dic))
         return dic
 
     def serializeForm(self):
@@ -866,17 +873,21 @@ class UserInterface(object):
         Note: Hidens are not serialized, they are ignored
 
         '''
+
+        # import inspect
+        # logger.debug('Caller is : {}'.format(inspect.stack()))
+
         arr = []
         for k, v in self._gui.iteritems():
             logger.debug('serializing Key: {0}/{1}'.format(k, v.value))
             if v.isType(gui.InputField.HIDDEN_TYPE) and v.isSerializable() is False:
-                logger.debug('Field {0} is not serializable'.format(k))
+                # logger.debug('Field {0} is not serializable'.format(k))
                 continue
             if v.isType(gui.InputField.INFO_TYPE):
-                logger.debug('Field {} is a dummy field and will not be serialized')
+                # logger.debug('Field {} is a dummy field and will not be serialized')
                 continue
             if v.isType(gui.InputField.EDITABLE_LIST) or v.isType(gui.InputField.MULTI_CHOICE_TYPE):
-                logger.debug('Serializing value {0}'.format(v.value))
+                # logger.debug('Serializing value {0}'.format(v.value))
                 val = '\001' + pickle.dumps(v.value)
             elif v.isType(gui.InputField.NUMERIC_TYPE):
                 val = six.text_type(int(v.num()))

server/src/uds/core/workers/ServiceCacheUpdater.py

@@ -84,7 +84,9 @@ class ServiceCacheUpdater(Job):
         for sp in whichNeedsCaching:
             sp.userServices.update()  # Cleans cached queries
             # If this deployedService don't have a publication active and needs it, ignore it
-            if sp.activePublication() is None and sp.service.getInstance().publicationType is not None:
+            spServiceInstance = sp.service.getInstance()
+
+            if sp.activePublication() is None and spServiceInstance.publicationType is not None:
                 logger.debug('{} Needs publication but do not have one, cache test ignored'.format(sp))
                 continue
             # If it has any running publication, do not generate cache anymore

server/src/uds/core/workers/UserServiceCleaner.py

@@ -33,7 +33,7 @@
 from __future__ import unicode_literals
 
 from django.db import transaction
-from uds.core.managers.UserServiceManager import UserServiceManager
+from uds.core import managers
 from uds.core.util.Config import GlobalConfig
 from uds.models import UserService, getSqlDatetime
 from uds.core.util.State import State
@@ -79,6 +79,7 @@ class UserServiceRemover(Job):
                                                 deployed_service__service__provider__maintenance_mode=False)[0:UserServiceRemover.removeAtOnce]
         for us in removables:
             try:
-                UserServiceManager.manager().remove(us)
+                if managers.userServiceManager().canRemoveServiceFromDeployedService(us.deployed_service) is True:
+                    managers.userServiceManager().remove(us)
             except Exception:
                 logger.exception('Exception removing user service')

server/src/uds/models/ManagedObjectModel.py

@@ -39,7 +39,7 @@ from uds.models.UUIDModel import UUIDModel
 
 import logging
 
-__updated__ = '2015-05-06'
+__updated__ = '2016-03-09'
 
 
 logger = logging.getLogger(__name__)
@@ -56,6 +56,8 @@ class ManagedObjectModel(UUIDModel):
     data = models.TextField(default='')
     comments = models.CharField(max_length=256)
 
+    _cachedInstance = None
+
     class Meta(UUIDModel.Meta):
         '''
         Defines this is an abstract clas
@@ -77,6 +79,8 @@ class ManagedObjectModel(UUIDModel):
         if values is None and self.data is not None and self.data != '':
             obj.unserialize(self.data)
 
+        self._cachedInstance = None  # Ensures returns correct value on getInstance
+
     def getInstance(self, values=None):
         '''
         Instantiates the object this record contains.
@@ -92,10 +96,17 @@ class ManagedObjectModel(UUIDModel):
         Notes:
             Can be overriden
         '''
+        if self._cachedInstance is not None and values is None:
+            logger.debug('Got cached instance instead of deserializing a new one for {}'.format(self.name))
+            return self._cachedInstance
+
         klass = self.getType()
         env = self.getEnvironment()
         obj = klass(env, values)
         self.deserialize(obj, values)
+
+        self._cachedInstance = obj
+
         return obj
 
     def getType(self):

server/src/uds/models/Service.py

@@ -48,7 +48,7 @@ from uds.models.Provider import Provider
 import logging
 
 
-__updated__ = '2016-02-26'
+__updated__ = '2016-03-09'
 
 
 logger = logging.getLogger(__name__)
@@ -103,12 +103,19 @@ class Service(ManagedObjectModel, TaggingMixin):
 
         Raises:
         '''
+        if self._cachedInstance is not None and values is None:
+            logger.debug('Got cached instance instead of deserializing a new one for {}'.format(self.name))
+            return self._cachedInstance
+
         prov = self.provider.getInstance()
         sType = prov.getServiceByType(self.data_type)
         env = self.getEnvironment()
-        s = sType(env, prov, values)
+        obj = sType(env, prov, values)
-        self.deserialize(s, values)
+        self.deserialize(obj, values)
-        return s
+
+        self._cachedInstance = obj
+
+        return obj
 
     def getType(self):
         '''

server/src/uds/services/OVirt/OVirtLinkedService.py

@@ -41,7 +41,7 @@ from uds.core.ui import gui
 
 import logging
 
-__updated__ = '2016-02-26'
+__updated__ = '2016-03-09'
 
 logger = logging.getLogger(__name__)
 
@@ -175,8 +175,8 @@ class OVirtLinkedService(Service):
         defvalue='1'  # Default value is the ID of the choicefield
     )
 
-    ov = gui.HiddenField()
+    ov = gui.HiddenField(value=None)
-    ev = gui.HiddenField()  # We need to keep the env so we can instantiate the Provider
+    ev = gui.HiddenField(value=None)  # We need to keep the env so we can instantiate the Provider
 
     def initialize(self, values):
         '''
@@ -196,9 +196,6 @@ class OVirtLinkedService(Service):
             if int(self.memoryGuaranteed.value) > int(self.memory.value):
                 self.memoryGuaranteed.value = self.memory.value
 
-        self.ov.value = self.parent().serialize()
-        self.ev.value = self.parent().env.key
-
     def initGui(self):
         '''
         Loads required values inside

server/src/uds/services/OpenStack/LiveDeployment.py

@@ -39,7 +39,7 @@ from . import openStack
 import pickle
 import logging
 
-__updated__ = '2016-03-07'
+__updated__ = '2016-03-09'
 
 
 logger = logging.getLogger(__name__)
@@ -66,8 +66,8 @@ class LiveDeployment(UserDeployment):
     _reason = ''
     _queue = None
 
-    # : Recheck every six seconds by default (for task methods)
+    # : Recheck every this seconds by default (for task methods)
-    suggestedTime = 6
+    suggestedTime = 20
 
     def initialize(self):
         self._name = ''

server/src/uds/services/OpenStack/LivePublication.py

@@ -41,7 +41,7 @@ import six
 import logging
 
 
-__updated__ = '2016-03-07'
+__updated__ = '2016-03-09'
 
 
 logger = logging.getLogger(__name__)
@@ -57,7 +57,7 @@ class LivePublication(Publication):
     _state = 'r'
     _destroyAfter = 'n'
 
-    suggestedTime = 2  # : Suggested recheck time if publication is unfinished in seconds
+    suggestedTime = 20  # : Suggested recheck time if publication is unfinished in seconds
 
     def initialize(self):
         '''

server/src/uds/services/OpenStack/LiveService.py

@@ -42,7 +42,7 @@ from uds.core.ui import gui
 import six
 import logging
 
-__updated__ = '2016-03-08'
+__updated__ = '2016-03-09'
 
 logger = logging.getLogger(__name__)
 
@@ -137,8 +137,8 @@ class LiveService(Service):
         required=True
     )
 
-    ov = gui.HiddenField()
+    ov = gui.HiddenField(value=None)
-    ev = gui.HiddenField()  # We need to keep the env so we can instantiate the Provider
+    ev = gui.HiddenField(value=None)  # We need to keep the env so we can instantiate the Provider
 
     def initialize(self, values):
         '''
@@ -154,8 +154,8 @@ class LiveService(Service):
             if self.baseName.value.isdigit():
                 raise Service.ValidationException(_('The machine name can\'t be only numbers'))
 
-        self.ov.value = self.parent().serialize()
+        # self.ov.value = self.parent().serialize()
-        self.ev.value = self.parent().env.key
+        # self.ev.value = self.parent().env.key
 
         self._api = None
 
@@ -173,6 +173,7 @@ class LiveService(Service):
 
         # So we can instantiate parent to get API
         logger.debug(self.parent().serialize())
+
         self.ov.setDefValue(self.parent().serialize())
         self.ev.setDefValue(self.parent().env.key)
 

server/src/uds/services/OpenStack/Provider.py

@@ -38,16 +38,13 @@ from django.utils.translation import ugettext_noop as _
 from uds.core.services import ServiceProvider
 from uds.core.ui import gui
 from uds.core.util import validators
-from defusedxml import minidom
 
 from .LiveService import LiveService
 from . import openStack
 
-
 import logging
-import six
 
-__updated__ = '2016-03-04'
+__updated__ = '2016-03-09'
 
 logger = logging.getLogger(__name__)
 
@@ -107,7 +104,12 @@ class Provider(ServiceProvider):
     domain = gui.TextField(length=64, label=_('Domain'), order=8, tooltip=_('Domain name (default is Default)'), required=True, defvalue='Default')
     username = gui.TextField(length=64, label=_('Username'), order=9, tooltip=_('User with valid privileges on OpenStack'), required=True, defvalue='admin')
     password = gui.PasswordField(lenth=32, label=_('Password'), order=10, tooltip=_('Password of the user of OpenStack'), required=True)
-    timeout = gui.NumericField(length=3, label=_('Timeout'), defvalue='10', order=99, tooltip=_('Timeout in seconds of connection to OpenStack'), required=True)
+
+    maxPreparingServices = gui.NumericField(length=3, label=_('Creation concurrency'), defvalue='10', minValue=1, maxValue=128, order=50, tooltip=_('Maximum number of concurrently removing VMs'), required=True)
+    maxRemovingServices = gui.NumericField(length=3, label=_('Removal concurrency'), defvalue='5', minValue=1, maxValue=128, order=51, tooltip=_('Maximum number of concurrently removing VMs'), required=True)
+
+    timeout = gui.NumericField(length=3, label=_('Timeout'), defvalue='10', minValue=1, maxValue=128, order=99, tooltip=_('Timeout in seconds of connection to OpenStack'), required=True)
+
 
     # tenant = gui.TextField(length=64, label=_('Project'), order=6, tooltip=_('Project (tenant) for this provider'), required=True, defvalue='')
     # region = gui.TextField(length=64, label=_('Region'), order=7, tooltip=_('Region for this provider'), required=True, defvalue='RegionOne')