burmatovan/openuds
1
0
Fork 0
forked from shaba/openuds
Code Issues Pull Requests Projects Releases Wiki Activity

added "global" stats counter to optimize graphs on dashboard

Browse Source
This commit is contained in:
Adolfo Gómez García 2021-05-19 11:45:40 +02:00
parent e87727b48f
commit e7d1df5ba3
3 changed files with 80 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
server/src/uds/REST/methods/system.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2019 Virtual Cable S.L.
# Copyright (c) 2014-2021 Virtual Cable S.L.U.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -12,7 +12,7 @@
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of Virtual Cable S.L. nor the names of its contributors
# * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
@ -38,16 +38,21 @@ import typing
from uds import models
from uds.core.util.model import processUuid
from uds.core.util.stats import counters
from uds.core.util.cache import Cache
from uds.core.util.state import State
from uds.REST import Handler, RequestError, ResponseError
from uds.core.util import permissions
from uds.REST import Handler, RequestError, ResponseError, AccessDenied
logger = logging.getLogger(__name__)
if typing.TYPE_CHECKING:
from django.db.models import Model
cache = Cache('StatsDispatcher')
# Enclosed methods under /system path
# Enclosed methods under /stats path
POINTS = 300
SINCE = 30 # Days, if higer values used, ensure mysql/mariadb has a bigger sort buffer
USE_MAX = True
@ -70,10 +75,9 @@ def getServicesPoolsCounters(
if not cachedValue:
if not servicePool:
us = models.ServicePool()
complete = True # Get all deployed services stats
us.id = -1 # Global stats
else:
us = servicePool
complete = False
val: typing.List[typing.Mapping[str, typing.Any]] = []
for x in counters.getCounters(
us,
@ -82,10 +86,11 @@ def getServicesPoolsCounters(
to=to,
max_intervals=POINTS,
use_max=USE_MAX,
all=complete,
all=False,
):
val.append({'stamp': x[0], 'value': int(x[1])})
if len(val) > 2:
logger.debug('val: %s', val)
if len(val) >= 2:
cache.put(cacheKey, codecs.encode(pickle.dumps(val), 'zip'), 600)
else:
val = [{'stamp': since, 'value': 0}, {'stamp': to, 'value': 0}]
@ -100,12 +105,16 @@ def getServicesPoolsCounters(
class System(Handler):
needs_admin = True
needs_admin = False
needs_staff = True
def get(self):
logger.debug('args: %s', self._args)
# Only allow admin user for global stats
if len(self._args) == 1:
if self._args[0] == 'overview': # System overview
if not self._user.is_admin:
raise AccessDenied()
users: int = models.User.objects.count()
groups: int = models.Group.objects.count()
services: int = models.Service.objects.count()
@ -127,12 +136,25 @@ class System(Handler):
'restrained_services_pools': restrained_services_pools,
}
if len(self._args) == 2:
if len(self._args) in (2, 3):
# Extract pool if provided
pool: typing.Optional[models.ServicePool] = None
if len(self._args) == 3:
try:
pool = models.ServicePool.objects.get(uuid=processUuid(self._args[3]))
except Exception:
pool = None
# If pool is None, needs admin also
if not pool and not self._user.is_admin:
raise AccessDenied()
# Check permission for pool..
if not permissions.checkPermissions(self._user, typing.cast('Model', pool), permissions.PERMISSION_READ):
raise AccessDenied()
if self._args[0] == 'stats':
if self._args[1] == 'assigned':
return getServicesPoolsCounters(None, counters.CT_ASSIGNED)
return getServicesPoolsCounters(pool, counters.CT_ASSIGNED)
if self._args[1] == 'inuse':
return getServicesPoolsCounters(None, counters.CT_INUSE)
return getServicesPoolsCounters(pool, counters.CT_INUSE)
raise RequestError('invalid request')

1
server/src/uds/core/util/permissions.py
View File

@ -32,7 +32,6 @@
"""
import logging
import typing
from uds.REST.methods.permissions import Permissions
from uds import models
from uds.core.util import ot

57
server/src/uds/core/workers/stats_collector.py
View File

@ -32,7 +32,7 @@
import logging
import typing
from uds.models import ServicePool, Authenticator
from uds.models import ServicePool, Authenticator, getSqlDatetime
from uds.core.util.state import State
from uds.core.util.stats import counters
from uds.core.managers import statsManager
@ -47,7 +47,7 @@ class DeployedServiceStatsCollector(Job):
This Job is responsible for collecting stats for every deployed service every ten minutes
"""
frecuency = 599 # Once every ten minutes, 601 is prime, 599 also is prime
frecuency = 30 # Once every ten minutes, 601 is prime, 599 also is prime
friendly_name = 'Deployed Service Stats'
def run(self):
@ -56,6 +56,9 @@ class DeployedServiceStatsCollector(Job):
servicePoolsToCheck: typing.Iterable[ServicePool] = ServicePool.objects.filter(
state=State.ACTIVE
).iterator()
stamp = getSqlDatetime()
# Global counters
totalAssigned, totalInUse, totalCached = 0, 0, 0
for servicePool in servicePoolsToCheck:
try:
fltr = servicePool.assignedUserServices().exclude(
@ -64,17 +67,33 @@ class DeployedServiceStatsCollector(Job):
assigned = fltr.count()
inUse = fltr.filter(in_use=True).count()
# Cached user services
cached = servicePool.cachedUserServices().exclude(
state__in=State.INFO_STATES
).count()
counters.addCounter(servicePool, counters.CT_ASSIGNED, assigned)
counters.addCounter(servicePool, counters.CT_INUSE, inUse)
counters.addCounter(servicePool, counters.CT_CACHED, cached)
cached = (
servicePool.cachedUserServices()
.exclude(state__in=State.INFO_STATES)
.count()
)
totalAssigned += assigned
totalInUse += inUse
totalCached += cached
counters.addCounter(
servicePool, counters.CT_ASSIGNED, assigned, stamp=stamp
)
counters.addCounter(servicePool, counters.CT_INUSE, inUse, stamp=stamp)
counters.addCounter(
servicePool, counters.CT_CACHED, cached, stamp=stamp
)
except Exception:
logger.exception(
'Getting counters for service pool %s', servicePool.name
)
# Store a global "fake pool" with all stats
sp = ServicePool()
sp.id = -1
counters.addCounter(sp, counters.CT_ASSIGNED, totalAssigned, stamp=stamp)
counters.addCounter(sp, counters.CT_INUSE, totalInUse, stamp=stamp)
counters.addCounter(sp, counters.CT_CACHED, totalCached, stamp=stamp)
totalUsers, totalAssigned, totalWithService = 0, 0, 0
for auth in Authenticator.objects.all():
fltr = auth.users.filter(userServices__isnull=False).exclude(
userServices__state__in=State.INFO_STATES
@ -82,14 +101,30 @@ class DeployedServiceStatsCollector(Job):
users = auth.users.all().count()
users_with_service = fltr.distinct().count()
number_assigned_services = fltr.count()
counters.addCounter(auth, counters.CT_AUTH_USERS, users)
# Global counters
totalUsers += users
totalAssigned += number_assigned_services
totalWithService += users_with_service
counters.addCounter(auth, counters.CT_AUTH_USERS, users, stamp=stamp)
counters.addCounter(
auth, counters.CT_AUTH_SERVICES, number_assigned_services
auth, counters.CT_AUTH_SERVICES, number_assigned_services, stamp=stamp
)
counters.addCounter(
auth, counters.CT_AUTH_USERS_WITH_SERVICES, users_with_service
auth,
counters.CT_AUTH_USERS_WITH_SERVICES,
users_with_service,
stamp=stamp,
)
au = Authenticator()
au.id = -1
counters.addCounter(au, counters.CT_AUTH_USERS, totalUsers, stamp=stamp)
counters.addCounter(au, counters.CT_AUTH_SERVICES, totalAssigned, stamp=stamp)
counters.addCounter(
au, counters.CT_AUTH_USERS_WITH_SERVICES, totalWithService, stamp=stamp
)
logger.debug('Done Deployed service stats collector')