burmatovan/openuds
1
0
Fork 0
forked from shaba/openuds
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v3.5'

Browse Source
This commit is contained in:
Adolfo Gómez García 2022-06-07 22:05:20 +02:00
commit 1fb8956679
8 changed files with 134 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
actor/src/udsactor/rest.py
View File

@ -131,7 +131,7 @@ class UDSApi: # pylint: disable=too-few-public-methods
headers=headers, headers=headers,
verify=self._validateCert, verify=self._validateCert,
timeout=TIMEOUT, timeout=TIMEOUT,
proxies=NO_PROXY proxies=NO_PROXY # type: ignore
if disableProxy if disableProxy
else None, # if not proxies wanted, enforce it else None, # if not proxies wanted, enforce it
) )

19
server/src/uds/core/ui/user_interface.py
View File

@ -528,7 +528,7 @@ class gui:
def date(self, min: bool = True) -> datetime.date: def date(self, min: bool = True) -> datetime.date:
""" """
Returns the date tis objecct represents Returns the date this object represents
Args: Args:
min (bool, optional): If true, in case of invalid date will return "min" date, else "max". Defaults to True. min (bool, optional): If true, in case of invalid date will return "min" date, else "max". Defaults to True.
@ -543,6 +543,23 @@ class gui:
except Exception: except Exception:
return datetime.date.min if min else datetime.date.max return datetime.date.min if min else datetime.date.max
def datetime(self, min: bool) -> datetime.datetime:
"""
Returns the date this object represents
Args:
min (bool, optional): If true, in case of invalid date will return "min" date, else "max". Defaults to True.
Returns:
datetime.date: the date that this object holds, or "min" | "max" on error
"""
try:
return datetime.datetime.strptime(
self.value, '%Y-%m-%d'
) # ISO Format
except Exception:
return datetime.datetime.min if min else datetime.datetime.max
def stamp(self) -> int: def stamp(self) -> int:
return int( return int(
time.mktime( time.mktime(

7
server/src/uds/models/stats_counters.py
View File

@ -118,11 +118,12 @@ class StatsCounters(models.Model):
# Max intervals, if present, will adjust interval (that are seconds) # Max intervals, if present, will adjust interval (that are seconds)
max_intervals = kwargs.get('max_intervals', 0) max_intervals = kwargs.get('max_intervals', 0)
if max_intervals > 0: if max_intervals > 0:
interval = max(interval, int(to - since) / max_intervals) count = q.count()
if max_intervals < count:
max_intervals = count
interval = int(to - since) / max_intervals
floor = getSqlFnc('FLOOR') floor = getSqlFnc('FLOOR')
ceil = getSqlFnc('CEIL')
avg = getSqlFnc('AVG')
if interval > 0: if interval > 0:
q = q.extra( q = q.extra(
select={ select={

56
server/src/uds/reports/auto/__init__.py
View File

@ -57,6 +57,7 @@ reportAutoModelDct: typing.Mapping[str, typing.Type[ReportAutoModel]] = { # typ
'Provider': models.Provider, 'Provider': models.Provider,
} }
class ReportAutoType(UserInterfaceType): class ReportAutoType(UserInterfaceType):
def __new__(cls, name, bases, attrs) -> 'ReportAutoType': def __new__(cls, name, bases, attrs) -> 'ReportAutoType':
# Add gui for elements... # Add gui for elements...
@ -114,7 +115,7 @@ class ReportAuto(Report, metaclass=ReportAutoType):
def getModel(self) -> typing.Type[ReportAutoModel]: # type: ignore def getModel(self) -> typing.Type[ReportAutoModel]: # type: ignore
data_source = self.data_source.split('.')[0] data_source = self.data_source.split('.')[0]
return reportAutoModelDct[data_source] return reportAutoModelDct[data_source]
def initGui(self): def initGui(self):
@ -141,3 +142,56 @@ class ReportAuto(Report, metaclass=ReportAutoType):
return {'hour': 1, 'day': 24, 'week': 24 * 7, 'month': 24 * 30}[ return {'hour': 1, 'day': 24, 'week': 24 * 7, 'month': 24 * 30}[
self.interval.value self.interval.value
] ]
def getIntervalsList(self) -> typing.List[typing.Tuple[datetime.datetime, datetime.datetime]]:
intervals: typing.List[typing.Tuple[datetime.datetime, datetime.datetime]] = []
# Convert start and end dates to datetime objects from date objects
start = datetime.datetime.combine(self.startingDate(), datetime.time.min)
to = datetime.datetime.combine(self.endingDate(), datetime.time.max)
while start < to:
if self.interval.value == 'hour':
intervals.append((start, start + datetime.timedelta(hours=1)))
start += datetime.timedelta(hours=1)
elif self.interval.value == 'day':
intervals.append((start, start + datetime.timedelta(days=1)))
start += datetime.timedelta(days=1)
elif self.interval.value == 'week':
intervals.append((start, start + datetime.timedelta(days=7)))
start += datetime.timedelta(days=7)
elif self.interval.value == 'month':
next = (start + datetime.timedelta(days=32)).replace(day=1)
intervals.append((start, next))
start = next
logger.info('Intervals: {0}'.format(intervals))
return intervals
def adjustDate(self, d: datetime.date, isEndingDate: bool) -> datetime.date:
if self.interval.value in ('hour', 'day'):
return d
elif self.interval.value == 'week':
return (d - datetime.timedelta(days=d.weekday())).replace()
elif self.interval.value == 'month':
if not isEndingDate:
return d.replace(day=1)
else:
return (d + datetime.timedelta(days=32)).replace(day=1) - datetime.timedelta(days=1)
else:
return d
def formatDatetimeAsString(self, d: datetime.date) -> str:
if self.interval.value in ('hour', 'day'):
return d.strftime('%Y-%b-%d %H:%M:%S')
elif self.interval.value == 'week':
return d.strftime('%Y-%b-%d')
elif self.interval.value == 'month':
return d.strftime('%Y-%b')
else:
return d.strftime('%Y-%b-%d %H:%M:%S')
def startingDate(self) -> datetime.date:
return self.adjustDate(self.date_start.date(), False)
def endingDate(self) -> datetime.date:
return self.adjustDate(self.date_end.date(), True)

96
server/src/uds/reports/stats/auth_stats.py
View File

@ -30,6 +30,7 @@
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
""" """
import logging import logging
import datetime
import typing import typing
from django.utils.translation import gettext, gettext_lazy as _ from django.utils.translation import gettext, gettext_lazy as _
@ -46,7 +47,7 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MAX_ELEMENTS = 10000 MAX_ELEMENTS = 10000
BIG_INTERVAL = 3600 * 24 * 30 * 12 # 12 months
class AuthenticatorsStats(StatsReportAuto): class AuthenticatorsStats(StatsReportAuto):
dates = 'range' dates = 'range'
@ -62,9 +63,6 @@ class AuthenticatorsStats(StatsReportAuto):
uuid = 'a5a43bc0-d543-11ea-af8f-af01fa65994e' uuid = 'a5a43bc0-d543-11ea-af8f-af01fa65994e'
def generate(self) -> typing.Any: def generate(self) -> typing.Any:
since = self.date_start.date()
to = self.date_end.date()
interval = self.getIntervalInHours() * 3600
stats = [] stats = []
for a in self.getModelItems(): for a in self.getModelItems():
@ -73,69 +71,55 @@ class AuthenticatorsStats(StatsReportAuto):
services = 0 services = 0
userServices = 0 userServices = 0
servicesCounterIter = iter( for i in self.getIntervalsList():
counters.getCounters( start = i[0]
typing.cast('models.Authenticator', a), end = i[1]
counters.CT_AUTH_SERVICES, data = [0, 0, 0]
since=since, # Get stats for interval
to=to, for counter in counters.getCounters(
interval=interval, typing.cast('models.Authenticator', a),
limit=MAX_ELEMENTS, counters.CT_AUTH_SERVICES,
use_max=True, since=start,
) to=end,
) interval=BIG_INTERVAL,
usersWithServicesCounterIter = iter( limit=MAX_ELEMENTS,
counters.getCounters( use_max=True,
):
data[0] += counter[1]
for counter in counters.getCounters(
typing.cast('models.Authenticator', a), typing.cast('models.Authenticator', a),
counters.CT_AUTH_USERS_WITH_SERVICES, counters.CT_AUTH_USERS_WITH_SERVICES,
since=since, since=start,
to=to, to=end,
interval=interval, interval=BIG_INTERVAL,
limit=MAX_ELEMENTS, limit=MAX_ELEMENTS,
use_max=True, use_max=True,
) ):
) data[1] += counter[1]
for userCounter in counters.getCounters( for counter in counters.getCounters(
typing.cast('models.Authenticator', a), typing.cast('models.Authenticator', a),
counters.CT_AUTH_USERS, counters.CT_AUTH_USERS,
since=since, since=start,
to=to, to=end,
interval=interval, interval=BIG_INTERVAL,
limit=MAX_ELEMENTS, limit=MAX_ELEMENTS,
use_max=True, use_max=True,
): ):
try: data[2] += counter[1]
while True:
servicesCounter = next(servicesCounterIter)
if servicesCounter[0] >= userCounter[0]:
break
if userCounter[0] == servicesCounter[0]:
services = servicesCounter[1]
except StopIteration:
pass
try:
while True:
uservicesCounter = next(usersWithServicesCounterIter)
if uservicesCounter[0] >= userCounter[0]:
break
if userCounter[0] == uservicesCounter[0]:
userServices = uservicesCounter[1]
except StopIteration:
pass
stats.append( stats.append(
{ {
'date': userCounter[0], 'date': self.formatDatetimeAsString(start),
'users': userCounter[1] or 0, 'users': data[0],
'services': services, 'services': data[1],
'user_services': userServices, 'user_services': data[2],
} }
) )
logger.debug('Report Data Done') logger.debug('Report Data Done')
return self.templateAsPDF( return self.templateAsPDF(
'uds/reports/stats/authenticator_stats.html', 'uds/reports/stats/authenticator_stats.html',
dct={'data': stats}, dct={'data': stats},
header=gettext('Users usage list'), header=ugettext('Users usage list'),
water=gettext('UDS Report of users usage'), water=ugettext('UDS Report of users usage'),
) )

35
server/src/uds/reports/stats/pools_performance.py
View File

@ -110,40 +110,23 @@ class PoolPerformanceReport(StatsReport):
) -> typing.Tuple[str, typing.List, typing.List]: # pylint: disable=too-many-locals ) -> typing.Tuple[str, typing.List, typing.List]: # pylint: disable=too-many-locals
start = self.startDate.stamp() start = self.startDate.stamp()
end = self.endDate.stamp() end = self.endDate.stamp()
if self.samplingPoints.num() < 2:
self.samplingPoints.value = (
self.endDate.date() - self.startDate.date()
).days
if self.samplingPoints.num() < 2: if self.samplingPoints.num() < 2:
self.samplingPoints.value = 2 self.samplingPoints.value = 2
if self.samplingPoints.num() > 32: if self.samplingPoints.num() > 128:
self.samplingPoints.value = 32 self.samplingPoints.value = 128
samplingPoints = self.samplingPoints.num() samplingPoints = self.samplingPoints.num()
pools = self.getPools()
if not pools:
raise Exception(_('Select at least a service pool for the report'))
logger.debug('Pools: %s', pools)
# x axis label format # x axis label format
if end - start > 3600 * 24 * 2: if end - start > 3600 * 24 * 2:
xLabelFormat = 'SHORT_DATE_FORMAT' xLabelFormat = 'SHORT_DATE_FORMAT'
else: else:
xLabelFormat = 'SHORT_DATETIME_FORMAT' xLabelFormat = 'SHORT_DATETIME_FORMAT'
# Generate samplings interval
samplingIntervals: typing.List[typing.Tuple[int, int]] = [] samplingIntervals: typing.List[typing.Tuple[int, int]] = []
prevVal = None samplingIntervalSeconds = (end - start) / samplingPoints
for val in range(start, end, int((end - start) / (samplingPoints + 1))): for i in range(samplingPoints):
if prevVal is None: samplingIntervals.append((int(start + i * samplingIntervalSeconds), int(start + (i + 1) * samplingIntervalSeconds)))
prevVal = val
continue
samplingIntervals.append((prevVal, val))
prevVal = val
# Store dataUsers for all pools # Store dataUsers for all pools
poolsData = [] poolsData = []
@ -151,11 +134,11 @@ class PoolPerformanceReport(StatsReport):
fld = StatsManager.manager().getEventFldFor('username') fld = StatsManager.manager().getEventFldFor('username')
reportData = [] reportData = []
for p in pools: for p in self.getPools():
dataUsers = [] dataUsers = []
dataAccesses = [] dataAccesses = []
for interval in samplingIntervals: for interval in samplingIntervals:
key = (interval[0] + interval[1]) / 2 key = (interval[0] + interval[1]) // 2
q = ( q = (
StatsManager.manager() StatsManager.manager()
.getEvents( .getEvents(
@ -177,9 +160,9 @@ class PoolPerformanceReport(StatsReport):
reportData.append( reportData.append(
{ {
'name': p[1], 'name': p[1],
'date': tools.timestampAsStr(interval[0], xLabelFormat) 'date': tools.timestampAsStr(interval[0], 'SHORT_DATETIME_FORMAT')
+ ' - ' + ' - '
+ tools.timestampAsStr(interval[1], xLabelFormat), + tools.timestampAsStr(interval[1], 'SHORT_DATETIME_FORMAT'),
'users': len(q), 'users': len(q),
'accesses': accesses, 'accesses': accesses,
} }

8
server/src/uds/reports/stats/pools_usage_day.py
View File

@ -85,7 +85,6 @@ class CountersPoolAssigned(StatsReport):
def getData(self) -> typing.List[typing.Dict[str, typing.Any]]: def getData(self) -> typing.List[typing.Dict[str, typing.Any]]:
# Generate the sampling intervals and get dataUsers from db # Generate the sampling intervals and get dataUsers from db
start = self.startDate.date() start = self.startDate.date()
end = self.startDate.date() + datetime.timedelta(days=1)
data = [] data = []
@ -102,15 +101,14 @@ class CountersPoolAssigned(StatsReport):
pool, pool,
counters.CT_ASSIGNED, counters.CT_ASSIGNED,
since=start, since=start,
to=end, to=start+datetime.timedelta(days=1),
max_intervals=24, intervals=3600,
use_max=True, use_max=True,
all=False, all=False,
): ):
hour = x[0].hour hour = x[0].hour
val = int(x[1]) val = int(x[1])
if hours[hour] < val: hours[hour] = max(hours[hour], val)
hours[hour] = val
data.append({'uuid': pool.uuid, 'name': pool.name, 'hours': hours}) data.append({'uuid': pool.uuid, 'name': pool.name, 'hours': hours})

19
server/src/uds/reports/stats/user_access.py
View File

@ -96,10 +96,6 @@ class StatsReportLogin(StatsReport):
def getRangeData(self) -> typing.Tuple[str, typing.List, typing.List]: def getRangeData(self) -> typing.Tuple[str, typing.List, typing.List]:
start = self.startDate.stamp() start = self.startDate.stamp()
end = self.endDate.stamp() end = self.endDate.stamp()
# if self.samplingPoints.num() < 8:
# self.samplingPoints.value = (
# self.endDate.date() - self.startDate.date()
# ).days
if self.samplingPoints.num() < 2: if self.samplingPoints.num() < 2:
self.samplingPoints.value = 2 self.samplingPoints.value = 2
if self.samplingPoints.num() > 128: if self.samplingPoints.num() > 128:
@ -121,7 +117,7 @@ class StatsReportLogin(StatsReport):
data = [] data = []
reportData = [] reportData = []
for interval in samplingIntervals: for interval in samplingIntervals:
key = (interval[0] + interval[1]) / 2 key = (interval[0] + interval[1]) // 2
val = ( val = (
StatsManager.manager() StatsManager.manager()
.getEvents( .getEvents(
@ -132,12 +128,12 @@ class StatsReportLogin(StatsReport):
) )
.count() .count()
) )
data.append((key, val)) # @UndefinedVariable data.append((key, val))
reportData.append( reportData.append(
{ {
'date': tools.timestampAsStr(interval[0], xLabelFormat) 'date': tools.timestampAsStr(interval[0], 'SHORT_DATETIME_FORMAT')
+ ' - ' + ' - '
+ tools.timestampAsStr(interval[1], xLabelFormat), + tools.timestampAsStr(interval[1], 'SHORT_DATETIME_FORMAT'),
'users': val, 'users': val,
} }
) )
@ -163,13 +159,6 @@ class StatsReportLogin(StatsReport):
return dataWeek, dataHour, dataWeekHour return dataWeek, dataHour, dataWeekHour
def generate(self): def generate(self):
# Sample query:
# 'SELECT *, count(*) as number, CEIL(stamp/(3600))*3600 as block'
# ' FROM {table}'
# ' WHERE event_type = 0 and stamp >= {start} and stamp <= {end}'
# ' GROUP BY CEIL(stamp/(3600))'
# ' ORDER BY block'
xLabelFormat, data, reportData = self.getRangeData() xLabelFormat, data, reportData = self.getRangeData()
# #