shaba/openuds
1
0
Fork 1
mirror of https://github.com/dkmstr/openuds.git synced 2024-12-23 17:34:17 +03:00
Code Issues Releases Wiki Activity

Adding support for acummulated stats counters

Browse Source
This commit is contained in:
Adolfo Gómez García 2022-11-14 14:11:57 +01:00
parent 31e8e5f0af
commit 139ca448ff
No known key found for this signature in database
GPG Key ID: DD1ABF20724CDA23
12 changed files with 140 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
server/src/tests/REST/test_system.py
View File

@ -28,12 +28,16 @@
"""
Author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import typing
import datetime
import logging
from uds.REST.handlers import AUTH_TOKEN_HEADER
from uds.REST.methods.actor_v3 import MANAGED, UNMANAGED, ALLOWED_FAILS
from uds.core.util.stats import counters
from ..utils import rest
from ..fixtures import stats_counters
logger = logging.getLogger(__name__)
@ -64,5 +68,24 @@ class SystemTest(rest.test.RESTTestCase):
self.assertEqual(json['restrained_services_pools'], 0)
def test_charts(self):
pass
def test_chart_pool(self):
# First, create fixtures for the pool
DAYS = 30
for pool in [self.user_service_managed, self.user_service_unmanaged]:
stats_counters.create_stats_interval_total(
id=pool.deployed_service.id,
counter_type=[counters.CT_ASSIGNED, counters.CT_INUSE, counters.CT_CACHED],
since=datetime.datetime.now() - datetime.timedelta(days=DAYS),
days=DAYS,
number_per_hour=6,
value=lambda x, y: (x % y) * 10,
owner_type=counters.OT_SERVICEPOOL
)
# Now, test (will fail if not logged in)
response = self.client.get('/uds/rest/system/stats/assigned')
self.assertEqual(response.status_code, 403)
# Login as admin
self.login()
response = self.client.get('/uds/rest/system/stats/assigned')
self.assertEqual(response.status_code, 200)

2
server/src/tests/core/workers/test_stats_c_acummulator.py
View File

@ -76,7 +76,7 @@ class StatsAcummulatorTest(UDSTestCase):
days=DAYS,
number_per_hour=NUMBER_PER_HOUR,
value=StatsFunction(10 ** (pool_id + 1)),
owner_type=counters.OT_DEPLOYED,
owner_type=counters.OT_SERVICEPOOL,
)
# Setup worker

2
server/src/tests/fixtures/stats_counters.py vendored
View File

@ -81,7 +81,7 @@ def create_stats_interval_total(
days: int,
number_per_hour: int,
value: typing.Union[int, typing.Callable[[int, int], int]],
owner_type: int = counters.OT_DEPLOYED,
owner_type: int = counters.OT_SERVICEPOOL,
) -> typing.List[models.StatsCounters]:
'''
Creates a list of counters with the given type, counter_type, since and to, save it in the database

2
server/src/uds/REST/methods/system.py
View File

@ -89,7 +89,7 @@ def getServicesPoolsCounters(
to=to,
max_intervals=POINTS,
use_max=USE_MAX,
all=False,
all=us.id == -1,
):
val.append({'stamp': x[0], 'value': int(x[1])})
logger.debug('val: %s', val)

4
server/src/uds/core/managers/log.py
View File

@ -48,7 +48,7 @@ logger = logging.getLogger(__name__)
(
OT_USERSERVICE,
OT_PUBLICATION,
OT_DEPLOYED_SERVICE,
OT_SERVICEPOOL,
OT_SERVICE,
OT_PROVIDER,
OT_USER,
@ -63,7 +63,7 @@ logger = logging.getLogger(__name__)
transDict: typing.Dict[typing.Type['Model'], int] = {
models.UserService: OT_USERSERVICE,
models.ServicePoolPublication: OT_PUBLICATION,
models.ServicePool: OT_DEPLOYED_SERVICE,
models.ServicePool: OT_SERVICEPOOL,
models.Service: OT_SERVICE,
models.Provider: OT_PROVIDER,
models.User: OT_USER,

70
server/src/uds/core/managers/stats.py
View File

@ -57,6 +57,14 @@ REVERSE_FLDS_EQUIV: typing.Mapping[str, str] = {
}
class AccumStat(typing.NamedTuple):
stamp: int
n: int # Number of elements in this interval
sum: int # Sum of elements in this interval
max: int # Max of elements in this interval
min: int # Min of elements in this interval
class StatsManager(metaclass=singleton.Singleton):
"""
Manager for statistics, so we can provide usefull info about platform usage
@ -73,7 +81,12 @@ class StatsManager(metaclass=singleton.Singleton):
def manager() -> 'StatsManager':
return StatsManager() # Singleton pattern will return always the same instance
def __doCleanup(self, model: typing.Type[typing.Union['StatsCounters', 'StatsEvents', 'StatsCountersAccum']]) -> None:
def __doCleanup(
self,
model: typing.Type[
typing.Union['StatsCounters', 'StatsEvents', 'StatsCountersAccum']
],
) -> None:
minTime = time.mktime(
(
getSqlDatetime()
@ -173,6 +186,61 @@ class StatsManager(metaclass=singleton.Singleton):
use_max=use_max,
)
def getAcumCounters(
self,
intervalType: StatsCountersAccum.IntervalType,
counterType: int,
owner_type: typing.Optional[int] = None,
owner_id: typing.Optional[int] = None,
since: typing.Optional[typing.Union[datetime.datetime, int]] = None,
points: typing.Optional[int] = None,
) -> typing.Generator[AccumStat, None, None]:
if since is None:
if points is None:
points = 100 # If since is not specified, we need at least points, get a default
since = getSqlDatetime() - datetime.timedelta(
seconds=intervalType.seconds() * points
)
if isinstance(since, datetime.datetime):
since = int(since.timestamp())
# Filter from since to now, get at most points
query = StatsCountersAccum.objects.filter(
interval_type=intervalType,
counter_type=counterType,
stamp__gte=since,
).order_by('stamp')[0:points]
if owner_type is not None:
query = query.filter(owner_type=owner_type)
if owner_id is not None:
query = query.filter(owner_id=owner_id)
# Create a numpy array with all data, stamp, n, sum, max, min (stamp, v_count,v_sum,v_max,v_min)
# Now, get exactly the points we need
stamp = since
last = AccumStat(stamp, 0, 0, 0, 0)
for rec in query:
# While query stamp is greater than stamp, repeat last AccumStat
while rec.stamp > stamp:
# Append to numpy array
yield last
stamp += intervalType.seconds()
last = last._replace(stamp=stamp) # adjust stamp
# Now, we have a record that is greater or equal to stamp, so we can use it
# but replace record stamp with stamp
last = AccumStat(
stamp,
rec.v_count,
rec.v_sum,
rec.v_max,
rec.v_min,
)
# Append to numpy array
yield last
stamp += intervalType.seconds()
def cleanupCounters(self):
"""
Removes all counters previous to configured max keep time for stat information from database.

39
server/src/uds/core/util/stats/counters.py
View File

@ -37,8 +37,15 @@ import typing
from django.utils.translation import gettext_lazy as _
from django.db.models import Model
from uds.core.managers.stats import StatsManager
from uds.models import NEVER, Provider, Service, ServicePool, Authenticator
from uds.core.managers.stats import StatsManager, AccumStat
from uds.models import (
NEVER,
Provider,
Service,
ServicePool,
Authenticator,
StatsCountersAccum,
)
logger = logging.getLogger(__name__)
@ -60,11 +67,11 @@ CounterClass = typing.TypeVar(
CT_CACHED,
) = range(8)
OT_PROVIDER, OT_SERVICE, OT_DEPLOYED, OT_AUTHENTICATOR = range(4)
OT_PROVIDER, OT_SERVICE, OT_SERVICEPOOL, OT_AUTHENTICATOR = range(4)
# Helpers
def _get_Id(obj):
return obj.id
return obj.id if obj.id != -1 else None
def _get_P_S_Ids(provider) -> typing.Tuple:
@ -82,7 +89,9 @@ def _get_P_S_DS_Ids(provider) -> typing.Tuple:
return res
idRetriever: typing.Mapping[typing.Type[Model], typing.Mapping[int, typing.Callable]] = {
idRetriever: typing.Mapping[
typing.Type[Model], typing.Mapping[int, typing.Callable]
] = {
Provider: {
CT_LOAD: _get_Id,
CT_STORAGE: _get_P_S_Ids,
@ -114,7 +123,7 @@ counterTypes: typing.Mapping[int, typing.Tuple[typing.Type[Model], ...]] = {
}
objectTypes: typing.Mapping[typing.Type[Model], int] = {
ServicePool: OT_DEPLOYED,
ServicePool: OT_SERVICEPOOL,
Service: OT_SERVICE,
Provider: OT_PROVIDER,
Authenticator: OT_AUTHENTICATOR,
@ -221,6 +230,24 @@ def getCounterTitle(counterType: int) -> str:
return titles.get(counterType, '').title()
def getAcumCounters(
intervalType: StatsCountersAccum.IntervalType,
counterType: int,
onwer_type: typing.Optional[int] = None,
owner_id: typing.Optional[int] = None,
since: typing.Optional[typing.Union[datetime.datetime, int]] = None,
points: typing.Optional[int] = None,
) -> typing.Generator[AccumStat, None, None]:
yield from StatsManager.manager().getAcumCounters(
intervalType=intervalType,
counterType=counterType,
owner_type=onwer_type,
owner_id=owner_id,
since=since,
points=points,
)
# Data initialization
def _initializeData() -> None:
"""

8
server/src/uds/core/util/stats/events.py
View File

@ -84,7 +84,7 @@ EVENT_NAMES: typing.Mapping[int, str] = {
(
OT_PROVIDER,
OT_SERVICE,
OT_DEPLOYED,
OT_SERVICEPOOL,
OT_AUTHENTICATOR,
OT_OSMANAGER
) = range(5)
@ -92,13 +92,13 @@ EVENT_NAMES: typing.Mapping[int, str] = {
TYPES_NAMES: typing.Mapping[int, str] = {
OT_PROVIDER: 'Provider',
OT_SERVICE: 'Service',
OT_DEPLOYED: 'Deployed',
OT_SERVICEPOOL: 'Deployed',
OT_AUTHENTICATOR: 'Authenticator',
OT_OSMANAGER: 'OS Manager'
}
MODEL_TO_EVENT: typing.Mapping[typing.Type['models.Model'], int] = {
ServicePool: OT_DEPLOYED,
ServicePool: OT_SERVICEPOOL,
Service: OT_SERVICE,
Provider: OT_PROVIDER,
Authenticator: OT_AUTHENTICATOR,
@ -161,7 +161,7 @@ def getOwner(ownerType: int, ownerId: int) -> typing.Optional['models.Model']:
return Provider.objects.get(pk=ownerId)
elif ownerType == OT_SERVICE:
return Service.objects.get(pk=ownerId)
elif ownerType == OT_DEPLOYED:
elif ownerType == OT_SERVICEPOOL:
return ServicePool.objects.get(pk=ownerId)
elif ownerType == OT_AUTHENTICATOR:
return Authenticator.objects.get(pk=ownerId)

3
server/src/uds/models/stats_counters.py
View File

@ -114,8 +114,7 @@ class StatsCounters(models.Model):
max_intervals = kwargs.get('max_intervals') or 0
if max_intervals > 0:
count = q.count()
if max_intervals < count:
max_intervals = count
max_intervals = max(min(max_intervals, count), 2)
interval = int(to - since) / max_intervals
floor = getSqlFnc('FLOOR')

2
server/src/uds/reports/stats/pool_users_summary.py
View File

@ -91,7 +91,7 @@ class UsageSummaryByUsersPool(StatsReport):
items = (
StatsManager.manager()
.getEvents(
events.OT_DEPLOYED,
events.OT_SERVICEPOOL,
(events.ET_LOGIN, events.ET_LOGOUT),
owner_id=pool.id,
since=start,

2
server/src/uds/reports/stats/pools_performance.py
View File

@ -142,7 +142,7 @@ class PoolPerformanceReport(StatsReport):
q = (
StatsManager.manager()
.getEvents(
events.OT_DEPLOYED,
events.OT_SERVICEPOOL,
events.ET_ACCESS,
since=interval[0],
to=interval[1],

2
server/src/uds/reports/stats/usage_by_pool.py
View File

@ -101,7 +101,7 @@ class UsageByPool(StatsReport):
items = (
StatsManager.manager()
.getEvents(
events.OT_DEPLOYED,
events.OT_SERVICEPOOL,
(events.ET_LOGIN, events.ET_LOGOUT),
owner_id=pool.id,
since=start,