diff --git a/server/src/tests/core/workers/test_stats_c_acummulator.py b/server/src/tests/core/workers/test_stats_c_acummulator.py index 4cd15caf6..208526cb4 100644 --- a/server/src/tests/core/workers/test_stats_c_acummulator.py +++ b/server/src/tests/core/workers/test_stats_c_acummulator.py @@ -41,6 +41,7 @@ from ...fixtures import stats_counters as fixtures_stats_counters from uds.core.workers import stats_collector from uds.core.environment import Environment +from uds.core.util import config START_DATE = datetime.datetime(2009, 12, 4, 0, 0, 0) @@ -79,7 +80,7 @@ class StatsAcummulatorTest(UDSTransactionTestCase): ) # Setup worker - stats_collector.STATS_ACCUM_MAX_CHUNK_TIME.set(DAYS // 2 + 1) + config.GlobalConfig.STATS_ACCUM_MAX_CHUNK_TIME.set(DAYS // 2 + 1) stats_collector.StatsAccumulator.setup() def test_stats_accumulator(self): diff --git a/server/src/uds/core/managers/stats.py b/server/src/uds/core/managers/stats.py index 0aa9695d5..52318fd45 100644 --- a/server/src/uds/core/managers/stats.py +++ b/server/src/uds/core/managers/stats.py @@ -73,7 +73,7 @@ class StatsManager(metaclass=singleton.Singleton): def manager() -> 'StatsManager': return StatsManager() # Singleton pattern will return always the same instance - def __doCleanup(self, model: typing.Type[typing.Union['StatsCounters', 'StatsEvents']]) -> None: + def __doCleanup(self, model: typing.Type[typing.Union['StatsCounters', 'StatsEvents', 'StatsCountersAccum']]) -> None: minTime = time.mktime( ( getSqlDatetime() @@ -178,6 +178,7 @@ class StatsManager(metaclass=singleton.Singleton): Removes all counters previous to configured max keep time for stat information from database. """ self.__doCleanup(StatsCounters) + self.__doCleanup(StatsCountersAccum) def getEventFldFor(self, fld: str) -> str: ''' diff --git a/server/src/uds/core/util/config.py b/server/src/uds/core/util/config.py index 07f12a59a..b22c3570f 100644 --- a/server/src/uds/core/util/config.py +++ b/server/src/uds/core/util/config.py @@ -559,6 +559,21 @@ class GlobalConfig: type=Config.NUMERIC_FIELD, help=_('Statistics duration, in days'), ) + # Statisctis accumulation frequency, in seconds + STATS_ACCUM_FREQUENCY: Config.Value = Config.section(GLOBAL_SECTION).value( + 'statsAccumFrequency', + '14400', + type=Config.NUMERIC_FIELD, + help=_('Frequency of stats collection in seconds. Default is 4 hours (14400 seconds)'), + ) + # Statisctis accumulation chunk size, in days + STATS_ACCUM_MAX_CHUNK_TIME = Config.section(GLOBAL_SECTION).value( + 'statsAccumMaxChunkTime', + '7', + type=Config.NUMERIC_FIELD, + help=_('Maximum number of time to accumulate on one run. Default is 7 (1 week)'), + ) + # If disallow login showing authenticatiors DISALLOW_GLOBAL_LOGIN: Config.Value = Config.section(GLOBAL_SECTION).value( 'disallowGlobalLogin', @@ -765,6 +780,7 @@ class GlobalConfig: for v in GlobalConfig.__dict__.values(): if isinstance(v, Config.Value): v.get() + logger.debug('Initialized global config value %s=%s', v.key(), v.get()) for c in _getLater: logger.debug('Get later: %s', c) diff --git a/server/src/uds/core/workers/stats_collector.py b/server/src/uds/core/workers/stats_collector.py index 6ac167e24..b94a2309c 100644 --- a/server/src/uds/core/workers/stats_collector.py +++ b/server/src/uds/core/workers/stats_collector.py @@ -44,19 +44,6 @@ from uds.core.util import config logger = logging.getLogger(__name__) -# Early declaration of config variable -STATS_ACCUM_FREQUENCY = config.Config.section(config.ADMIN_SECTION).value( - 'Stats Accumulation Frequency', - '14400', - type=config.Config.NUMERIC_FIELD, - help=_('Frequency of stats collection in seconds. Default is 4 hours (14400 seconds)'), -) -STATS_ACCUM_MAX_CHUNK_TIME = config.Config.section(config.ADMIN_SECTION).value( - 'Stats Accumulation Chunk', - '7', - type=config.Config.NUMERIC_FIELD, - help=_('Maximum number of time to accumulate on one run. Default is 7 (1 week)'), -) class DeployedServiceStatsCollector(Job): """ @@ -179,13 +166,13 @@ class StatsAccumulator(Job): """ frecuency = 3600 # Executed every 4 hours frecuency_cfg = ( - STATS_ACCUM_FREQUENCY + config.GlobalConfig.STATS_ACCUM_FREQUENCY ) friendly_name = 'Statistics acummulator' def run(self): try: - StatsManager.manager().acummulate(STATS_ACCUM_MAX_CHUNK_TIME.getInt()) + StatsManager.manager().acummulate(config.GlobalConfig.STATS_ACCUM_MAX_CHUNK_TIME.getInt()) except Exception as e: logger.exception('Compressing counters') diff --git a/server/src/uds/migrations/0043_auto_20220704_2120.py b/server/src/uds/migrations/0043_auto_20220704_2120.py index 13d67af93..573fed489 100644 --- a/server/src/uds/migrations/0043_auto_20220704_2120.py +++ b/server/src/uds/migrations/0043_auto_20220704_2120.py @@ -1,6 +1,8 @@ # Generated by Django 3.2.10 on 2022-07-04 21:20 from django.db import migrations, models +from django.db import connection + import django.db.models.deletion import uds.models.stats_counters_accum @@ -8,21 +10,20 @@ import uds.models.stats_counters_accum # InnoDB is tremendlously slow when using this table def forwards(apps, schema_editor): try: - from django.db import connection - # If we are not using MySQL, do nothing if connection.vendor != 'mysql': return cursor = connection.cursor() - # Check current table type, if it is not InnoDB, do nothing - cursor.execute( - 'SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = "uds_stats_c"' - ) - if cursor.fetchone()[0] == 'InnoDB': # type: ignore + tables_to_change = ['uds_stats_c', 'uds_stats_c_accum'] + for table in tables_to_change: + # Check current table type, if it is not InnoDB, do nothing cursor.execute( - 'ALTER TABLE uds_stats_c ENGINE=MyISAM' + 'SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s', + [table], ) + if cursor.fetchone()[0] == 'InnoDB': # type: ignore + cursor.execute(f'ALTER TABLE {table} ENGINE=MyISAM') except Exception: # nosec: fine pass @@ -97,7 +98,7 @@ class Migration(migrations.Migration): ('owner_id', models.IntegerField(default=0)), ('owner_type', models.SmallIntegerField(default=0)), ('counter_type', models.SmallIntegerField(default=0)), - ('interval_type', models.SmallIntegerField(choices=[(2, 'HOUR'), (3, 'DAY')], default=uds.models.stats_counters_accum.StatsCountersAccum.IntervalType['HOUR'])), + ('interval_type', models.SmallIntegerField(choices=[(1, 'HOUR'), (2, 'DAY')], default=uds.models.stats_counters_accum.StatsCountersAccum.IntervalType['HOUR'])), ('stamp', models.IntegerField(default=0)), ('v_count', models.IntegerField(default=0)), ('v_sum', models.IntegerField(default=0)), diff --git a/server/src/uds/models/stats_counters_accum.py b/server/src/uds/models/stats_counters_accum.py index 27395ee39..4a4e7a430 100644 --- a/server/src/uds/models/stats_counters_accum.py +++ b/server/src/uds/models/stats_counters_accum.py @@ -216,7 +216,7 @@ class StatsCountersAccum(models.Model): ) """Stores accumulated data in StatsCountersAccum""" - # Acummulate data + # Acummulate data, only register if there is data accumulated: typing.List[StatsCountersAccum] = [ StatsCountersAccum( owner_type=rec['owner_type'], @@ -230,12 +230,14 @@ class StatsCountersAccum(models.Model): v_min=rec['min'], v_max=rec['max'], ) - for rec in query + for rec in query if rec['sum'] and rec['min'] and rec['max'] ] logger.debug('Inserting %s records', len(accumulated)) - # If we have more than 20 inserts, do it in a single query - StatsCountersAccum.objects.bulk_create(accumulated) + # Insert in chunks of 2500 records + while accumulated: + StatsCountersAccum.objects.bulk_create(accumulated[:2500]) + accumulated = accumulated[2500:] def __str__(self) -> str: return f'{datetime.datetime.fromtimestamp(self.stamp)} - {self.owner_type}:{self.owner_id}:{self.counter_type} {StatsCountersAccum.IntervalType(self.interval_type)} {self.v_count},{self.v_sum},{self.v_min},{self.v_max}'