diff --git a/client/administration/installer/UDSAdminInstaller/MSChart.exe b/client/administration/installer/UDSAdminInstaller/MSChart.exe deleted file mode 100644 index 5ca98b85..00000000 Binary files a/client/administration/installer/UDSAdminInstaller/MSChart.exe and /dev/null differ diff --git a/client/administration/installer/UDSAdminInstaller/RedistLicenseMSChart.docx b/client/administration/installer/UDSAdminInstaller/RedistLicenseMSChart.docx deleted file mode 100644 index bb2798c7..00000000 Binary files a/client/administration/installer/UDSAdminInstaller/RedistLicenseMSChart.docx and /dev/null differ diff --git a/client/administration/installer/UDSAdminInstaller/mschart.txt b/client/administration/installer/UDSAdminInstaller/mschart.txt new file mode 100644 index 00000000..33c0ee9e --- /dev/null +++ b/client/administration/installer/UDSAdminInstaller/mschart.txt @@ -0,0 +1,5 @@ +To build installed, mschart.exe is needed (provided by Microsoft) + +You can get it at: + +http://www.microsoft.com/en-us/download/details.aspx?id=14422 diff --git a/server/.settings/org.eclipse.core.resources.prefs b/server/.settings/org.eclipse.core.resources.prefs index 46e83e65..214bf5eb 100644 --- a/server/.settings/org.eclipse.core.resources.prefs +++ b/server/.settings/org.eclipse.core.resources.prefs @@ -98,6 +98,7 @@ encoding//src/uds/core/workers/PublicationCleaner.py=utf-8 encoding//src/uds/core/workers/ServiceCacheUpdater.py=utf-8 encoding//src/uds/core/workers/StatsCollector.py=utf-8 encoding//src/uds/core/workers/UserServiceCleaner.py=utf-8 +encoding//src/uds/core/workers/__init__.py=utf-8 encoding//src/uds/dispatchers/__init__.py=utf-8 encoding//src/uds/dispatchers/pam/urls.py=utf-8 encoding//src/uds/dispatchers/pam/views.py=utf-8 diff --git a/server/src/uds/core/__init__.py b/server/src/uds/core/__init__.py index 7447ff0d..5b88ba9a 100644 --- a/server/src/uds/core/__init__.py +++ b/server/src/uds/core/__init__.py @@ -41,8 +41,3 @@ import services import auths import transports -def __init__(): - from managers.TaskManager import TaskManager - TaskManager.registerScheduledTasks() - -__init__() \ No newline at end of file diff --git a/server/src/uds/core/jobs/Job.py b/server/src/uds/core/jobs/Job.py index 6b47096c..e5a8bfeb 100644 --- a/server/src/uds/core/jobs/Job.py +++ b/server/src/uds/core/jobs/Job.py @@ -31,7 +31,6 @@ @author: Adolfo Gómez, dkmaster at dkmon dot com ''' -from uds.models import Scheduler from uds.core import Environmentable import logging @@ -40,7 +39,8 @@ logger = logging.getLogger(__name__) class Job(Environmentable): # Default frecuency, once a day. Remenber that precision will be based on "granurality" of Scheduler # If a job is used for delayed execution, this attribute is in fact ignored - frecuency = Scheduler.DAY + frecuency = 24*3600+3 + friendly_name = 'Unknown' def __init__(self, environment): ''' diff --git a/server/src/uds/core/managers/StatsManager.py b/server/src/uds/core/managers/StatsManager.py index b99238a8..4fc5c485 100644 --- a/server/src/uds/core/managers/StatsManager.py +++ b/server/src/uds/core/managers/StatsManager.py @@ -30,17 +30,13 @@ @author: Adolfo Gómez, dkmaster at dkmon dot com ''' -from uds.models import Provider -from uds.models import Service +from uds.core.util.Config import GlobalConfig import logging logger = logging.getLogger(__name__) - - - class StatsManager(object): ''' Manager for loggins statistics, so we can provide usefull info about platform usage @@ -138,11 +134,27 @@ class StatsManager(object): return StatsCounters.get_grouped(ownerType, counterType, owner_id = ownerIds, since = since, to = to, limit = limit, use_max = use_max) - def cleanupCounter(self): + def cleanupCounters(self): ''' Removes all counters previous to configured max keep time for stat information from database. ''' - pass + from uds.models import StatsCounters, getSqlDatetime, optimizeTable + from django.db import connection, transaction + import datetime + import time + + minTime = time.mktime( (getSqlDatetime() - datetime.timedelta(days = GlobalConfig.STATS_DURATION.getInt())).timetuple() ) + dbTable = StatsCounters.__dict__['_meta'].db_table + + # Don't like how django executes this (recovers all IDS and lauches "DELETE .. WHERE id IN ...) + #StatsCounters.objects.filter(stamp__lt=minTime).delete() + # Used dict, cause environment says _meta is not known :-) + query = 'DELETE FROM {0} where STAMP < {1}'.format(dbTable, minTime) + cursor = connection.cursor() + cursor.execute(query) + # This table will hold a big amount of data, and mayby we erase a a big number of records also. + # This will ensure table is in "good" shape (testing right now, will see at future) + optimizeTable(dbTable) # Event stats # Counter stats diff --git a/server/src/uds/core/managers/TaskManager.py b/server/src/uds/core/managers/TaskManager.py index 8eb16dbe..81576a38 100644 --- a/server/src/uds/core/managers/TaskManager.py +++ b/server/src/uds/core/managers/TaskManager.py @@ -77,33 +77,23 @@ class TaskManager(object): @staticmethod def registerScheduledTasks(): - from uds.core.workers.ServiceCacheUpdater import ServiceCacheUpdater - from uds.core.workers.UserServiceCleaner import UserServiceInfoItemsCleaner, UserServiceRemover - from uds.core.workers.PublicationCleaner import PublicationInfoItemsCleaner, PublicationCleaner - from uds.core.workers.CacheCleaner import CacheCleaner - from uds.core.workers.DeployedServiceCleaner import DeployedServiceInfoItemsCleaner, DeployedServiceRemover - from uds.core.workers.StatsCollector import DeployedServiceStatsCollector logger.info("Registering sheduled tasks") - TaskManager.registerJob('Service Cache Updater', ServiceCacheUpdater) - TaskManager.registerJob('User Service Info Cleaner', UserServiceInfoItemsCleaner) - TaskManager.registerJob('User Service Cleaner', UserServiceRemover) - TaskManager.registerJob('Publications Info Cleaner', PublicationInfoItemsCleaner) - TaskManager.registerJob('Publication Cleaner', PublicationCleaner) - TaskManager.registerJob('Utility Cache Cleaner', CacheCleaner) - TaskManager.registerJob('Deployed Service Info Cleaner', DeployedServiceInfoItemsCleaner) - TaskManager.registerJob('Deployed Service Cleaner', DeployedServiceRemover) - TaskManager.registerJob('Deployed Service Stats', DeployedServiceStatsCollector) - + from uds.core import workers + @staticmethod def run(): TaskManager.keepRunning = True # Runs Scheduler in a separate thread and DelayedTasks here + TaskManager.registerScheduledTasks() + noSchedulers = GlobalConfig.SCHEDULER_THREADS.getInt() noDelayedTasks = GlobalConfig.DELAYED_TASKS_THREADS.getInt() + logger.info('Starting {0} schedulers and {1} task executors'.format(noSchedulers, noDelayedTasks)) + threads = [] for n in range(noSchedulers): thread = SchedulerThread() diff --git a/server/src/uds/core/util/Config.py b/server/src/uds/core/util/Config.py index 627c068b..31dfdf18 100644 --- a/server/src/uds/core/util/Config.py +++ b/server/src/uds/core/util/Config.py @@ -41,12 +41,12 @@ logger = logging.getLogger(__name__) GLOBAL_SECTION = 'UDS' -class Config: +class Config(object): ''' Keeps persistend configuration data ''' - class _Value: + class _Value(object): def __init__(self, section, key, default = '', crypt = False, longText = False): self._section = section self._key = key @@ -162,7 +162,7 @@ class Config: except Exception: pass -class GlobalConfig: +class GlobalConfig(object): ''' Simple helper to keep track of global configuration ''' @@ -170,9 +170,9 @@ class GlobalConfig: # Delay between cache checks. reducing this number will increase cache generation speed but also will load service providers CACHE_CHECK_DELAY = Config.section(GLOBAL_SECTION).value('cacheCheckDelay', '19') # Delayed task number of threads PER SERVER, with higher number of threads, deplayed task will complete sooner, but it will give more load to overall system - DELAYED_TASKS_THREADS = Config.section(GLOBAL_SECTION).value('delayedTasksThreads', '2') + DELAYED_TASKS_THREADS = Config.section(GLOBAL_SECTION).value('delayedTasksThreads', '4') # Number of scheduler threads running PER SERVER, with higher number of threads, deplayed task will complete sooner, but it will give more load to overall system - SCHEDULER_THREADS = Config.section(GLOBAL_SECTION).value('schedulerThreads', '2') + SCHEDULER_THREADS = Config.section(GLOBAL_SECTION).value('schedulerThreads', '3') # Waiting time before removing "errored" and "removed" publications, cache, and user assigned machines. Time is in seconds CLEANUP_CHECK = Config.section(GLOBAL_SECTION).value('cleanupCheck', '3607') # Time to maintaing "info state" items before removing it, in seconds @@ -223,37 +223,19 @@ class GlobalConfig: # Time to restrain a deployed service in case it gives some error at some point RESTRAINT_TIME = Config.section(GLOBAL_SECTION).value('restrainTime', '600') + # Statistics duration, in days + STATS_DURATION = Config.section(GLOBAL_SECTION).value('statsDuration', '365') + initDone = False @staticmethod def initialize(): try: + # All configurations are upper case # Tries to initialize database data for global config so it is stored asap and get cached for use - GlobalConfig.SESSION_EXPIRE_TIME.get() - GlobalConfig.CACHE_CHECK_DELAY.get() - GlobalConfig.DELAYED_TASKS_THREADS.get() - GlobalConfig.SCHEDULER_THREADS.get() - GlobalConfig.CLEANUP_CHECK.get() - GlobalConfig.KEEP_INFO_TIME.get() - GlobalConfig.MAX_PREPARING_SERVICES.get() - GlobalConfig.MAX_REMOVING_SERVICES.get() - GlobalConfig.USER_SERVICE_CLEAN_NUMBER.get() - GlobalConfig.REMOVAL_CHECK.get() - GlobalConfig.LOGIN_URL.get() - GlobalConfig.USER_SESSION_LENGTH.get() - GlobalConfig.SUPER_USER_LOGIN.get() - GlobalConfig.SUPER_USER_PASS.get() - GlobalConfig.ADMIN_IDLE_TIME.get() - GlobalConfig.CHECK_UNUSED_TIME.get() - GlobalConfig.CSS.get() - GlobalConfig.MAX_LOGIN_TRIES.get() - GlobalConfig.LOGIN_BLOCK.get() - GlobalConfig.AUTORUN_SERVICE.get() - GlobalConfig.REDIRECT_TO_HTTPS.get() - GlobalConfig.MAX_INITIALIZING_TIME.get() - GlobalConfig.CUSTOM_HTML_LOGIN.get() - GlobalConfig.MAX_LOGS_PER_ELEMENT.get() - GlobalConfig.RESTRAINT_TIME.get() + for v in GlobalConfig.__dict__.itervalues(): + if type(v) is Config._Value: + v.get() except: logger.debug('Config table do not exists!!!, maybe we are installing? :-)') diff --git a/server/src/uds/core/workers/AssignedAndUnused.py b/server/src/uds/core/workers/AssignedAndUnused.py index 5be83d6c..64635789 100644 --- a/server/src/uds/core/workers/AssignedAndUnused.py +++ b/server/src/uds/core/workers/AssignedAndUnused.py @@ -40,8 +40,9 @@ import logging logger = logging.getLogger(__name__) -class AssignedAndUnused(Job): +class AssignedAndUnused(object): # When derived from Job, it will be auto-registered frecuency = GlobalConfig.CHECK_UNUSED_TIME.getInt() + friendly_name = 'Unused services checker' def __init__(self, environment): super(AssignedAndUnused,self).__init__(environment) @@ -50,7 +51,7 @@ class AssignedAndUnused(Job): for ds in DeployedService.objects.all(): osm = ds.osmanager.getInstance() if osm.processUnusedMachines is True: - logger.debug('Processing unused machines for {0}'.format(osm)) + logger.debug('Processing unused services for {0}'.format(osm)) since_state = getSqlDatetime() - timedelta( seconds = GlobalConfig.CHECK_UNUSED_TIME.getInt() / 2 ) for us in ds.assignedUserServices().select_for_update().filter(in_use=False,since_state__lt=since_state): logger.debug('Found unused assigned service {0}'.format(us)) diff --git a/server/src/uds/core/workers/CacheCleaner.py b/server/src/uds/core/workers/CacheCleaner.py index fc6fb90f..ad351b63 100644 --- a/server/src/uds/core/workers/CacheCleaner.py +++ b/server/src/uds/core/workers/CacheCleaner.py @@ -41,6 +41,7 @@ logger = logging.getLogger(__name__) class CacheCleaner(Job): frecuency = 3600*24 # Once a day + friendly_name = 'Utility Cache Cleaner' def __init__(self, environment): super(CacheCleaner,self).__init__(environment) diff --git a/server/src/uds/core/workers/DeployedServiceCleaner.py b/server/src/uds/core/workers/DeployedServiceCleaner.py index 2e26ac48..7ecb69ed 100644 --- a/server/src/uds/core/workers/DeployedServiceCleaner.py +++ b/server/src/uds/core/workers/DeployedServiceCleaner.py @@ -43,6 +43,7 @@ logger = logging.getLogger(__name__) class DeployedServiceInfoItemsCleaner(Job): frecuency = GlobalConfig.CLEANUP_CHECK.getInt() # Request run cache "info" cleaner every configured seconds. If config value is changed, it will be used at next reload + friendly_name = 'Deployed Service Info Cleaner' def __init__(self, environment): super(DeployedServiceInfoItemsCleaner,self).__init__(environment) @@ -54,6 +55,7 @@ class DeployedServiceInfoItemsCleaner(Job): class DeployedServiceRemover(Job): frecuency = GlobalConfig.REMOVAL_CHECK.getInt() # Request run publication "removal" every configued seconds. If config value is changed, it will be used at next reload + friendly_name = 'Deployed Service Cleaner' def __init__(self, environment): super(DeployedServiceRemover,self).__init__(environment) diff --git a/server/src/uds/core/workers/PublicationCleaner.py b/server/src/uds/core/workers/PublicationCleaner.py index a3e123b9..3644ed90 100644 --- a/server/src/uds/core/workers/PublicationCleaner.py +++ b/server/src/uds/core/workers/PublicationCleaner.py @@ -45,6 +45,7 @@ logger = logging.getLogger(__name__) class PublicationInfoItemsCleaner(Job): frecuency = GlobalConfig.CLEANUP_CHECK.getInt() # Request run cache "info" cleaner every configured seconds. If config value is changed, it will be used at next reload + friendly_name = 'Publications Info Cleaner' def __init__(self, environment): super(PublicationInfoItemsCleaner,self).__init__(environment) @@ -55,6 +56,7 @@ class PublicationInfoItemsCleaner(Job): class PublicationCleaner(Job): frecuency = GlobalConfig.REMOVAL_CHECK.getInt() # Request run publication "removal" every configued seconds. If config value is changed, it will be used at next reload + friendly_name = 'Publication Cleaner' def __init__(self, environment): super(PublicationCleaner,self).__init__(environment) diff --git a/server/src/uds/core/workers/ServiceCacheUpdater.py b/server/src/uds/core/workers/ServiceCacheUpdater.py index 22a40ce2..0b6e34e9 100644 --- a/server/src/uds/core/workers/ServiceCacheUpdater.py +++ b/server/src/uds/core/workers/ServiceCacheUpdater.py @@ -54,6 +54,7 @@ class ServiceCacheUpdater(Job): ''' frecuency = GlobalConfig.CACHE_CHECK_DELAY.getInt() # Request run cache manager every configured seconds (defaults to 20 seconds). # If config value is changed, it will be used at next reload + friendly_name = 'Service Cache Updater' def __init__(self, environment): super(ServiceCacheUpdater,self).__init__(environment) diff --git a/server/src/uds/core/workers/StatsCollector.py b/server/src/uds/core/workers/StatsCollector.py index eb0838be..44d3409a 100644 --- a/server/src/uds/core/workers/StatsCollector.py +++ b/server/src/uds/core/workers/StatsCollector.py @@ -33,6 +33,7 @@ from uds.models import DeployedService from uds.core.util.State import State from uds.core.util.stats import counters +from uds.core.managers import StatsManager from uds.core.jobs.Job import Job import logging @@ -43,6 +44,7 @@ logger = logging.getLogger(__name__) class DeployedServiceStatsCollector(Job): frecuency = 599 # Once every ten minutes, 601 is prime + friendly_name = 'Deployed Service Stats' def __init__(self, environment): super(DeployedServiceStatsCollector,self).__init__(environment) @@ -64,4 +66,15 @@ class DeployedServiceStatsCollector(Job): logger.debug('Done Deployed service stats collector') - \ No newline at end of file +class StatsCleaner(Job): + + frecuency = 3600*24*15 # Ejecuted just once every 15 days + friendly_name = 'Statistic housekeeping' + + def run(self): + logger.debug('Starting statistics cleanup') + try: + StatsManager().cleanupCounters() + except: + logger.exception('Cleaning up counters') + logger.debug('Donde statistics cleanup') diff --git a/server/src/uds/core/workers/UserServiceCleaner.py b/server/src/uds/core/workers/UserServiceCleaner.py index 6247831b..d3c0c94d 100644 --- a/server/src/uds/core/workers/UserServiceCleaner.py +++ b/server/src/uds/core/workers/UserServiceCleaner.py @@ -50,6 +50,7 @@ logger = logging.getLogger(__name__) class UserServiceInfoItemsCleaner(Job): frecuency = GlobalConfig.CLEANUP_CHECK.getInt() # Request run cache "info" cleaner every configured seconds. If config value is changed, it will be used at next reload + friendly_name = 'User Service Info Cleaner' def __init__(self, environment): super(UserServiceInfoItemsCleaner,self).__init__(environment) @@ -63,6 +64,8 @@ class UserServiceInfoItemsCleaner(Job): class UserServiceRemover(Job): frecuency = GlobalConfig.REMOVAL_CHECK.getInt() # Request run cache "info" cleaner every configued seconds. If config value is changed, it will be used at next reload + friendly_name = 'User Service Cleaner' + removeAtOnce = GlobalConfig.USER_SERVICE_CLEAN_NUMBER.getInt() # Same, it will work at reload def __init__(self, environment): diff --git a/server/src/uds/core/workers/__init__.py b/server/src/uds/core/workers/__init__.py index e69de29b..f2f6bb37 100644 --- a/server/src/uds/core/workers/__init__.py +++ b/server/src/uds/core/workers/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# +# Copyright (c) 2012 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' +.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com +''' + +def __init__(): + ''' + This imports all packages that are descendant of this package, and, after that, + it register all subclases of service provider as + ''' + import os.path, pkgutil + import sys + from uds.core import jobs + from uds.core.managers.TaskManager import TaskManager + + # Dinamycally import children of this package. + pkgpath = os.path.dirname(sys.modules[__name__].__file__) + for _, name, _ in pkgutil.iter_modules([pkgpath]): + __import__(name, globals(), locals(), [], -1) + + p = jobs.Job + # This is marked as error in IDE, but it's not (__subclasses__) + for cls in p.__subclasses__(): + # Limit to autoregister just workers jobs inside this module + if cls.__module__[0:16] == 'uds.core.workers': + TaskManager.registerJob(cls.friendly_name, cls) + +__init__() diff --git a/server/src/uds/models.py b/server/src/uds/models.py index 17b1048e..b44e0dd9 100644 --- a/server/src/uds/models.py +++ b/server/src/uds/models.py @@ -61,9 +61,9 @@ def getSqlDatetime(unix=False): * sqlite ''' from django.db import connection - con = connection - cursor = con.cursor() - if con.vendor == 'mysql': + cursor = connection.cursor() + + if connection.vendor == 'mysql': cursor.execute('SELECT NOW()') date = cursor.fetchone()[0] else: @@ -74,6 +74,12 @@ def getSqlDatetime(unix=False): else: return date +def optimizeTable(dbTable): + from django.db import connection + cursor = connection.cursor() + + if connection.vendor == 'mysql': + cursor.execute('OPTIMIZE TABLE {0}'.format(dbTable)) # Services class Provider(models.Model):