Merge remote-tracking branch 'origin/v3.5'

This commit is contained in:
Adolfo Gómez García 2022-06-07 22:05:20 +02:00
commit 1fb8956679
8 changed files with 134 additions and 108 deletions

View File

@ -131,7 +131,7 @@ class UDSApi: # pylint: disable=too-few-public-methods
headers=headers,
verify=self._validateCert,
timeout=TIMEOUT,
proxies=NO_PROXY
proxies=NO_PROXY # type: ignore
if disableProxy
else None, # if not proxies wanted, enforce it
)

View File

@ -528,7 +528,7 @@ class gui:
def date(self, min: bool = True) -> datetime.date:
"""
Returns the date tis objecct represents
Returns the date this object represents
Args:
min (bool, optional): If true, in case of invalid date will return "min" date, else "max". Defaults to True.
@ -543,6 +543,23 @@ class gui:
except Exception:
return datetime.date.min if min else datetime.date.max
def datetime(self, min: bool) -> datetime.datetime:
"""
Returns the date this object represents
Args:
min (bool, optional): If true, in case of invalid date will return "min" date, else "max". Defaults to True.
Returns:
datetime.date: the date that this object holds, or "min" | "max" on error
"""
try:
return datetime.datetime.strptime(
self.value, '%Y-%m-%d'
) # ISO Format
except Exception:
return datetime.datetime.min if min else datetime.datetime.max
def stamp(self) -> int:
return int(
time.mktime(

View File

@ -118,11 +118,12 @@ class StatsCounters(models.Model):
# Max intervals, if present, will adjust interval (that are seconds)
max_intervals = kwargs.get('max_intervals', 0)
if max_intervals > 0:
interval = max(interval, int(to - since) / max_intervals)
count = q.count()
if max_intervals < count:
max_intervals = count
interval = int(to - since) / max_intervals
floor = getSqlFnc('FLOOR')
ceil = getSqlFnc('CEIL')
avg = getSqlFnc('AVG')
if interval > 0:
q = q.extra(
select={

View File

@ -57,6 +57,7 @@ reportAutoModelDct: typing.Mapping[str, typing.Type[ReportAutoModel]] = { # typ
'Provider': models.Provider,
}
class ReportAutoType(UserInterfaceType):
def __new__(cls, name, bases, attrs) -> 'ReportAutoType':
# Add gui for elements...
@ -114,7 +115,7 @@ class ReportAuto(Report, metaclass=ReportAutoType):
def getModel(self) -> typing.Type[ReportAutoModel]: # type: ignore
data_source = self.data_source.split('.')[0]
return reportAutoModelDct[data_source]
def initGui(self):
@ -141,3 +142,56 @@ class ReportAuto(Report, metaclass=ReportAutoType):
return {'hour': 1, 'day': 24, 'week': 24 * 7, 'month': 24 * 30}[
self.interval.value
]
def getIntervalsList(self) -> typing.List[typing.Tuple[datetime.datetime, datetime.datetime]]:
intervals: typing.List[typing.Tuple[datetime.datetime, datetime.datetime]] = []
# Convert start and end dates to datetime objects from date objects
start = datetime.datetime.combine(self.startingDate(), datetime.time.min)
to = datetime.datetime.combine(self.endingDate(), datetime.time.max)
while start < to:
if self.interval.value == 'hour':
intervals.append((start, start + datetime.timedelta(hours=1)))
start += datetime.timedelta(hours=1)
elif self.interval.value == 'day':
intervals.append((start, start + datetime.timedelta(days=1)))
start += datetime.timedelta(days=1)
elif self.interval.value == 'week':
intervals.append((start, start + datetime.timedelta(days=7)))
start += datetime.timedelta(days=7)
elif self.interval.value == 'month':
next = (start + datetime.timedelta(days=32)).replace(day=1)
intervals.append((start, next))
start = next
logger.info('Intervals: {0}'.format(intervals))
return intervals
def adjustDate(self, d: datetime.date, isEndingDate: bool) -> datetime.date:
if self.interval.value in ('hour', 'day'):
return d
elif self.interval.value == 'week':
return (d - datetime.timedelta(days=d.weekday())).replace()
elif self.interval.value == 'month':
if not isEndingDate:
return d.replace(day=1)
else:
return (d + datetime.timedelta(days=32)).replace(day=1) - datetime.timedelta(days=1)
else:
return d
def formatDatetimeAsString(self, d: datetime.date) -> str:
if self.interval.value in ('hour', 'day'):
return d.strftime('%Y-%b-%d %H:%M:%S')
elif self.interval.value == 'week':
return d.strftime('%Y-%b-%d')
elif self.interval.value == 'month':
return d.strftime('%Y-%b')
else:
return d.strftime('%Y-%b-%d %H:%M:%S')
def startingDate(self) -> datetime.date:
return self.adjustDate(self.date_start.date(), False)
def endingDate(self) -> datetime.date:
return self.adjustDate(self.date_end.date(), True)

View File

@ -30,6 +30,7 @@
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
import logging
import datetime
import typing
from django.utils.translation import gettext, gettext_lazy as _
@ -46,7 +47,7 @@ if typing.TYPE_CHECKING:
logger = logging.getLogger(__name__)
MAX_ELEMENTS = 10000
BIG_INTERVAL = 3600 * 24 * 30 * 12 # 12 months
class AuthenticatorsStats(StatsReportAuto):
dates = 'range'
@ -62,9 +63,6 @@ class AuthenticatorsStats(StatsReportAuto):
uuid = 'a5a43bc0-d543-11ea-af8f-af01fa65994e'
def generate(self) -> typing.Any:
since = self.date_start.date()
to = self.date_end.date()
interval = self.getIntervalInHours() * 3600
stats = []
for a in self.getModelItems():
@ -73,69 +71,55 @@ class AuthenticatorsStats(StatsReportAuto):
services = 0
userServices = 0
servicesCounterIter = iter(
counters.getCounters(
typing.cast('models.Authenticator', a),
counters.CT_AUTH_SERVICES,
since=since,
to=to,
interval=interval,
limit=MAX_ELEMENTS,
use_max=True,
)
)
usersWithServicesCounterIter = iter(
counters.getCounters(
for i in self.getIntervalsList():
start = i[0]
end = i[1]
data = [0, 0, 0]
# Get stats for interval
for counter in counters.getCounters(
typing.cast('models.Authenticator', a),
counters.CT_AUTH_SERVICES,
since=start,
to=end,
interval=BIG_INTERVAL,
limit=MAX_ELEMENTS,
use_max=True,
):
data[0] += counter[1]
for counter in counters.getCounters(
typing.cast('models.Authenticator', a),
counters.CT_AUTH_USERS_WITH_SERVICES,
since=since,
to=to,
interval=interval,
since=start,
to=end,
interval=BIG_INTERVAL,
limit=MAX_ELEMENTS,
use_max=True,
)
)
for userCounter in counters.getCounters(
typing.cast('models.Authenticator', a),
counters.CT_AUTH_USERS,
since=since,
to=to,
interval=interval,
limit=MAX_ELEMENTS,
use_max=True,
):
try:
while True:
servicesCounter = next(servicesCounterIter)
if servicesCounter[0] >= userCounter[0]:
break
if userCounter[0] == servicesCounter[0]:
services = servicesCounter[1]
except StopIteration:
pass
try:
while True:
uservicesCounter = next(usersWithServicesCounterIter)
if uservicesCounter[0] >= userCounter[0]:
break
if userCounter[0] == uservicesCounter[0]:
userServices = uservicesCounter[1]
except StopIteration:
pass
):
data[1] += counter[1]
for counter in counters.getCounters(
typing.cast('models.Authenticator', a),
counters.CT_AUTH_USERS,
since=start,
to=end,
interval=BIG_INTERVAL,
limit=MAX_ELEMENTS,
use_max=True,
):
data[2] += counter[1]
stats.append(
{
'date': userCounter[0],
'users': userCounter[1] or 0,
'services': services,
'user_services': userServices,
'date': self.formatDatetimeAsString(start),
'users': data[0],
'services': data[1],
'user_services': data[2],
}
)
logger.debug('Report Data Done')
return self.templateAsPDF(
'uds/reports/stats/authenticator_stats.html',
dct={'data': stats},
header=gettext('Users usage list'),
water=gettext('UDS Report of users usage'),
header=ugettext('Users usage list'),
water=ugettext('UDS Report of users usage'),
)

View File

@ -110,40 +110,23 @@ class PoolPerformanceReport(StatsReport):
) -> typing.Tuple[str, typing.List, typing.List]: # pylint: disable=too-many-locals
start = self.startDate.stamp()
end = self.endDate.stamp()
if self.samplingPoints.num() < 2:
self.samplingPoints.value = (
self.endDate.date() - self.startDate.date()
).days
if self.samplingPoints.num() < 2:
self.samplingPoints.value = 2
if self.samplingPoints.num() > 32:
self.samplingPoints.value = 32
if self.samplingPoints.num() > 128:
self.samplingPoints.value = 128
samplingPoints = self.samplingPoints.num()
pools = self.getPools()
if not pools:
raise Exception(_('Select at least a service pool for the report'))
logger.debug('Pools: %s', pools)
# x axis label format
if end - start > 3600 * 24 * 2:
xLabelFormat = 'SHORT_DATE_FORMAT'
else:
xLabelFormat = 'SHORT_DATETIME_FORMAT'
# Generate samplings interval
samplingIntervals: typing.List[typing.Tuple[int, int]] = []
prevVal = None
for val in range(start, end, int((end - start) / (samplingPoints + 1))):
if prevVal is None:
prevVal = val
continue
samplingIntervals.append((prevVal, val))
prevVal = val
samplingIntervalSeconds = (end - start) / samplingPoints
for i in range(samplingPoints):
samplingIntervals.append((int(start + i * samplingIntervalSeconds), int(start + (i + 1) * samplingIntervalSeconds)))
# Store dataUsers for all pools
poolsData = []
@ -151,11 +134,11 @@ class PoolPerformanceReport(StatsReport):
fld = StatsManager.manager().getEventFldFor('username')
reportData = []
for p in pools:
for p in self.getPools():
dataUsers = []
dataAccesses = []
for interval in samplingIntervals:
key = (interval[0] + interval[1]) / 2
key = (interval[0] + interval[1]) // 2
q = (
StatsManager.manager()
.getEvents(
@ -177,9 +160,9 @@ class PoolPerformanceReport(StatsReport):
reportData.append(
{
'name': p[1],
'date': tools.timestampAsStr(interval[0], xLabelFormat)
'date': tools.timestampAsStr(interval[0], 'SHORT_DATETIME_FORMAT')
+ ' - '
+ tools.timestampAsStr(interval[1], xLabelFormat),
+ tools.timestampAsStr(interval[1], 'SHORT_DATETIME_FORMAT'),
'users': len(q),
'accesses': accesses,
}

View File

@ -85,7 +85,6 @@ class CountersPoolAssigned(StatsReport):
def getData(self) -> typing.List[typing.Dict[str, typing.Any]]:
# Generate the sampling intervals and get dataUsers from db
start = self.startDate.date()
end = self.startDate.date() + datetime.timedelta(days=1)
data = []
@ -102,15 +101,14 @@ class CountersPoolAssigned(StatsReport):
pool,
counters.CT_ASSIGNED,
since=start,
to=end,
max_intervals=24,
to=start+datetime.timedelta(days=1),
intervals=3600,
use_max=True,
all=False,
):
hour = x[0].hour
val = int(x[1])
if hours[hour] < val:
hours[hour] = val
hours[hour] = max(hours[hour], val)
data.append({'uuid': pool.uuid, 'name': pool.name, 'hours': hours})

View File

@ -96,10 +96,6 @@ class StatsReportLogin(StatsReport):
def getRangeData(self) -> typing.Tuple[str, typing.List, typing.List]:
start = self.startDate.stamp()
end = self.endDate.stamp()
# if self.samplingPoints.num() < 8:
# self.samplingPoints.value = (
# self.endDate.date() - self.startDate.date()
# ).days
if self.samplingPoints.num() < 2:
self.samplingPoints.value = 2
if self.samplingPoints.num() > 128:
@ -121,7 +117,7 @@ class StatsReportLogin(StatsReport):
data = []
reportData = []
for interval in samplingIntervals:
key = (interval[0] + interval[1]) / 2
key = (interval[0] + interval[1]) // 2
val = (
StatsManager.manager()
.getEvents(
@ -132,12 +128,12 @@ class StatsReportLogin(StatsReport):
)
.count()
)
data.append((key, val)) # @UndefinedVariable
data.append((key, val))
reportData.append(
{
'date': tools.timestampAsStr(interval[0], xLabelFormat)
'date': tools.timestampAsStr(interval[0], 'SHORT_DATETIME_FORMAT')
+ ' - '
+ tools.timestampAsStr(interval[1], xLabelFormat),
+ tools.timestampAsStr(interval[1], 'SHORT_DATETIME_FORMAT'),
'users': val,
}
)
@ -163,13 +159,6 @@ class StatsReportLogin(StatsReport):
return dataWeek, dataHour, dataWeekHour
def generate(self):
# Sample query:
# 'SELECT *, count(*) as number, CEIL(stamp/(3600))*3600 as block'
# ' FROM {table}'
# ' WHERE event_type = 0 and stamp >= {start} and stamp <= {end}'
# ' GROUP BY CEIL(stamp/(3600))'
# ' ORDER BY block'
xLabelFormat, data, reportData = self.getRangeData()
#