From 44456213f784bab46acd0e6e793767ca2b8afc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20G=C3=B3mez=20Garc=C3=ADa?= Date: Mon, 3 Aug 2020 13:04:51 +0200 Subject: [PATCH] Added new report "Authenticators stats" --- server/src/uds/core/ui/__init__.py | 2 +- server/src/uds/core/ui/user_interface.py | 17 ++- server/src/uds/models/stats_counters.py | 13 +- server/src/uds/reports/__init__.py | 10 +- server/src/uds/reports/auto/__init__.py | 131 ++++++++++++++++++ server/src/uds/reports/auto/fields.py | 119 ++++++++++++++++ server/src/uds/reports/stats/__init__.py | 1 + server/src/uds/reports/stats/auth_stats.py | 110 +++++++++++++++ server/src/uds/reports/stats/base.py | 4 + .../reports/stats/authenticator_stats.html | 40 ++++++ 10 files changed, 433 insertions(+), 14 deletions(-) create mode 100644 server/src/uds/reports/auto/__init__.py create mode 100644 server/src/uds/reports/auto/fields.py create mode 100644 server/src/uds/reports/stats/auth_stats.py create mode 100644 server/src/uds/templates/uds/reports/stats/authenticator_stats.html diff --git a/server/src/uds/core/ui/__init__.py b/server/src/uds/core/ui/__init__.py index a37f97910..01c706abf 100644 --- a/server/src/uds/core/ui/__init__.py +++ b/server/src/uds/core/ui/__init__.py @@ -5,4 +5,4 @@ This module contains the definition of UserInterface, needed to describe the int between an UDS module and the administration interface """ -from .user_interface import gui, UserInterface +from .user_interface import gui, UserInterface, UserInterfaceType diff --git a/server/src/uds/core/ui/user_interface.py b/server/src/uds/core/ui/user_interface.py index 16abe27ff..bd8e53a12 100644 --- a/server/src/uds/core/ui/user_interface.py +++ b/server/src/uds/core/ui/user_interface.py @@ -103,16 +103,17 @@ class gui: # Helpers @staticmethod - def convertToChoices(vals: typing.Iterable[str]) -> typing.List[typing.Dict[str, str]]: + def convertToChoices(vals: typing.Union[typing.List[str], typing.MutableMapping[str, str]]) -> typing.List[typing.Dict[str, str]]: """ Helper to convert from array of strings to the same dict used in choice, multichoice, .. The id is set to values in the array (strings), while text is left empty. """ - res = [] - for v in vals: - res.append({'id': v, 'text': ''}) - return res + if isinstance(vals, (list, tuple)): + return [{'id': v, 'text': ''} for v in vals] + + # Dictionary + return [{'id': k, 'text': v} for k, v in vals.items()] @staticmethod def convertToList(vals: typing.Iterable[str]) -> typing.List[str]: @@ -672,6 +673,8 @@ class gui: def __init__(self, **options): super().__init__(**options) + if options.get('values') and isinstance(options.get('values'), dict): + options['values'] = gui.convertToChoices(options['values']) self._data['values'] = options.get('values', []) if 'fills' in options: # Save fnc to register as callback @@ -738,6 +741,8 @@ class gui: def __init__(self, **options): super().__init__(**options) + if options.get('values') and isinstance(options.get('values'), dict): + options['values'] = gui.convertToChoices(options['values']) self._data['values'] = options.get('values', []) self._data['rows'] = options.get('rows', -1) self._type(gui.InputField.MULTI_CHOICE_TYPE) @@ -826,7 +831,7 @@ class UserInterfaceType(type): return type.__new__(cls, classname, bases, newClassDict) -class UserInterface(metaclass=UserInterfaceType): +class UserInterface(metaclass=UserInterfaceType): """ This class provides the management for gui descriptions (user forms) diff --git a/server/src/uds/models/stats_counters.py b/server/src/uds/models/stats_counters.py index 67245e0bc..c3df68409 100644 --- a/server/src/uds/models/stats_counters.py +++ b/server/src/uds/models/stats_counters.py @@ -130,12 +130,17 @@ class StatsCounters(models.Model): if limit: filt += ' LIMIT {}'.format(limit) - fnc = getSqlFnc('MAX' if kwargs.get('use_max', False) else 'AVG') + if kwargs.get('use_max', False): + fnc = getSqlFnc('MAX') + ('(value)') + else: + fnc = getSqlFnc('CEIL') + '({}(value))'.format(getSqlFnc('AVG')) + + # fnc = getSqlFnc('MAX' if kwargs.get('use_max', False) else 'AVG') query = ( - 'SELECT -1 as id,-1 as owner_id,-1 as owner_type,-1 as counter_type, ' + stampValue + '*{}'.format(interval) + ' AS stamp,' + - getSqlFnc('CEIL') + '({0}(value)) AS value ' - 'FROM {1} WHERE {2}' + 'SELECT -1 as id,-1 as owner_id,-1 as owner_type,-1 as counter_type, ' + stampValue + '*{}'.format(interval) + ' AS stamp, ' + + '{} AS value ' + 'FROM {} WHERE {}' ).format(fnc, StatsCounters._meta.db_table, filt) logger.debug('Stats query: %s', query) diff --git a/server/src/uds/reports/__init__.py b/server/src/uds/reports/__init__.py index 85fc9e3b4..9846ae17f 100644 --- a/server/src/uds/reports/__init__.py +++ b/server/src/uds/reports/__init__.py @@ -57,12 +57,15 @@ def __init__(): """ This imports all packages that are descendant of this package, and, after that, """ + alreadyAdded: typing.Set[str] = set() + def addReportCls(cls: typing.Type[reports.Report]): logger.debug('Adding report %s', cls) availableReports.append(cls) def recursiveAdd(reportClass: typing.Type[reports.Report]): - if reportClass.uuid: + if reportClass.uuid and reportClass.uuid not in alreadyAdded: + alreadyAdded.add(reportClass.uuid) addReportCls(reportClass) else: logger.debug('Report class %s not added because it lacks of uuid (it is probably a base class)', reportClass) @@ -77,8 +80,9 @@ def __init__(): # __import__(name, globals(), locals(), [], 1) importlib.import_module('.' + name, __name__) # Local import - importlib.invalidate_caches() - recursiveAdd(reports.Report) + importlib.invalidate_caches() + + __init__() diff --git a/server/src/uds/reports/auto/__init__.py b/server/src/uds/reports/auto/__init__.py new file mode 100644 index 000000000..22c46713e --- /dev/null +++ b/server/src/uds/reports/auto/__init__.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- + +# +# Copyright (c) 2015-2019 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com +""" +import logging +import datetime +import typing + +from django.utils.translation import ugettext, ugettext_noop as _ + +from uds.core.ui import UserInterface, UserInterfaceType, gui +from uds.core.reports import Report +from uds import models + +from . import fields + +logger = logging.getLogger(__name__) + +ReportAutoModel = typing.TypeVar('ReportAutoModel', models.Authenticator, models.ServicePool, models.Service, models.Provider) + +class ReportAutoType(UserInterfaceType): + def __new__(cls, name, bases, attrs) -> 'ReportAutoType': + # Add gui for elements... + order = 1 + + # Check what source + if attrs.get('data_source'): + + attrs['source'] = fields.source_field(order, attrs['data_source'], attrs['multiple']) + order += 1 + + # Check if date must be added + if attrs.get('dates') == 'single': + attrs['date_start'] = fields.single_date_field(order) + order += 1 + + if attrs.get('dates') == 'range': + attrs['date_start'] = fields.start_date_field(order) + order += 1 + attrs['date_end'] = fields.end_date_field(order) + order += 1 + + # Check if data interval should be included + if attrs.get('intervals'): + attrs['interval'] = fields.intervals_field(order) + order += 1 + + return UserInterfaceType.__new__(cls, name, bases, attrs) + +class ReportAuto(Report, metaclass=ReportAutoType): + # Variables that will be overwriten on new class creation + source: typing.ClassVar[typing.Union[gui.MultiChoiceField, gui.ChoiceField]] + date_start: typing.ClassVar[gui.DateField] + date_end: typing.ClassVar[gui.DateField] + interval: typing.ClassVar[gui.ChoiceField] + + # Dates can be None, 'single' or 'range' to auto add date fields + dates: typing.ClassVar[typing.Optional[str]] = None + intervals: bool = False + # Valid data_source: + # * ServicePool.usage + # * ServicePool.assigned + # * Authenticator.users + # * Authenticator.services + # * Authenticator.user_with_services + data_source: str = '' + + # If True, will allow selection of multiple "source" elements + multiple: bool = False + + def getModel(self) -> typing.Type[ReportAutoModel]: + data_source = self.data_source.split('.')[0] + + return typing.cast(ReportAutoModel, { + 'ServicePool': models.ServicePool, + 'Authenticator': models.Authenticator, + 'Service': models.Service, + 'Provider': models.Provider + }[data_source]) + + def initGui(self): + # Fills datasource + fields.source_field_data(self.getModel(), self.data_source, self.source) + + def getModelItems(self) -> typing.Iterable[ReportAutoModel]: + model = self.getModel() + + filters = [self.source.value] if isinstance(self.source, gui.ChoiceField) else self.source.value + + if '0-0-0-0' in filters: + items = model.objects.all() + else: + items = model.objects.filter(uuid__in=filters) + + return items + + def getIntervalInHours(self): + return { + 'hour': 1, + 'day': 24, + 'week': 24*7, + 'month': 24*30 + }[self.interval.value] diff --git a/server/src/uds/reports/auto/fields.py b/server/src/uds/reports/auto/fields.py new file mode 100644 index 000000000..b0cf449dd --- /dev/null +++ b/server/src/uds/reports/auto/fields.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +# +# Copyright (c) 2020 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com +""" +import datetime +import logging +import typing + +from django.utils.translation import ugettext_noop as _ + +from uds.core.ui import gui +from uds import models + +logger = logging.getLogger(__name__) + +def start_date_field(order:int) -> gui.DateField: + return gui.DateField( + order=order, + label=_('Starting date'), + tooltip=_('Starting date for report'), + defvalue=datetime.date.min, + required=True + ) + +def single_date_field(order: int) -> gui.DateField: + return gui.DateField( + order=order, + label=_('Date'), + tooltip=_('Date for report'), + defvalue=datetime.date.today(), + required=True + ) + + +def end_date_field(order: int) -> gui.DateField: + return gui.DateField( + order=order, + label=_('Ending date'), + tooltip=_('ending date for report'), + defvalue=datetime.date.max, + required=True + ) + +def intervals_field(order: int) -> gui.ChoiceField: + return gui.ChoiceField( + label=_('Report data interval'), + order=order, + values={ + 'hour': _('Hourly'), + 'day': _('Daily'), + 'week': _('Weekly'), + 'month': _('Monthly') + }, + tooltip=_('Interval for report data'), + required=True, + defvalue='day' + ) + +def source_field(order: int, data_source: str, multiple: bool) -> typing.Union[gui.ChoiceField, gui.MultiChoiceField, None]: + if not data_source: + return None + + data_source = data_source.split('.')[0] + logger.debug('SOURCE: %s', data_source) + + fieldType: typing.Type = gui.ChoiceField if not multiple else gui.MultiChoiceField + + labels: typing.Any = { + 'ServicePool': (_('Service pool'), _('Service pool for report')), + 'Authenticator': (_('Authenticator'), _('Authenticator for report')), + 'Service': (_('Service'), _('Service for report')), + 'Provider': (_('Service provider'), _('Service provider for report')), + }.get(data_source) + + logger.debug('Labels: %s, %s', labels, fieldType) + + return fieldType( + label=labels[0], + order=order, + tooltip=labels[1], + required=True + ) + +def source_field_data(model: typing.Any, data_source: str, field: typing.Union[gui.ChoiceField, gui.MultiChoiceField]) -> None: + + dataList: typing.List[gui.ChoiceType] = [{'id': x.uuid, 'text': x.name} for x in model.objects.all().order_by('name')] + + if isinstance(field, gui.MultiChoiceField): + dataList.insert(0, {'id': '0-0-0-0', 'text': _('All')}) + + field.setValues(dataList) diff --git a/server/src/uds/reports/stats/__init__.py b/server/src/uds/reports/stats/__init__.py index 53b43d194..2da76ae38 100644 --- a/server/src/uds/reports/stats/__init__.py +++ b/server/src/uds/reports/stats/__init__.py @@ -40,3 +40,4 @@ from . import usage_by_pool from . import pool_users_summary from . import pools_usage_summary +from . import auth_stats diff --git a/server/src/uds/reports/stats/auth_stats.py b/server/src/uds/reports/stats/auth_stats.py new file mode 100644 index 000000000..c2fdb18d5 --- /dev/null +++ b/server/src/uds/reports/stats/auth_stats.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +# +# Copyright (c) 2015-2019 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com +""" +import io +import csv +import datetime +import logging +import typing + +from django.utils.translation import ugettext, ugettext_lazy as _ + +from uds.core.ui import gui +from uds.core.util.stats import counters + +from .base import StatsReportAuto + +logger = logging.getLogger(__name__) + +MAX_ELEMENTS = 10000 + +class AuthenticatorsStats(StatsReportAuto): + dates = 'range' + intervals = True + data_source = 'Authenticator' + multiple = True + + filename = 'auths_stats.pdf' + name = _('Statistics by authenticator') # Report name + description = _('Generates a report with the statistics of an authenticator for a desired period') # Report description + uuid = 'a5a43bc0-d543-11ea-af8f-af01fa65994e' + + def generate(self) -> typing.Any: + since = self.date_start.date() + to = self.date_end.date() + interval = self.getIntervalInHours() * 3600 + + stats = [] + for a in self.getModelItems(): + # Will show a.name on every change... + stats.append({'date': a.name, 'users': None}) + + services = 0 + userServices = 0 + servicesCounterIter = iter(counters.getCounters(a, counters.CT_AUTH_SERVICES, since=since, interval=interval, limit=MAX_ELEMENTS, use_max=True)) + usersWithServicesCounterIter = iter(counters.getCounters(a, counters.CT_AUTH_USERS_WITH_SERVICES, since=since, interval=interval, limit=MAX_ELEMENTS, use_max=True)) + for userCounter in counters.getCounters(a, counters.CT_AUTH_USERS, since=since, interval=interval, limit=MAX_ELEMENTS, use_max=True): + try: + while True: + servicesCounter = next(servicesCounterIter) + if servicesCounter[0] >= userCounter[0]: + break + if userCounter[0] == servicesCounter[0]: + services = servicesCounter[1] + except StopIteration: + pass + + try: + while True: + uservicesCounter = next(usersWithServicesCounterIter) + if uservicesCounter[0] >= userCounter[0]: + break + if userCounter[0] == uservicesCounter[0]: + userServices = uservicesCounter[1] + except StopIteration: + pass + + stats.append({ + 'date': userCounter[0], + 'users': userCounter[1] or 0, + 'services': services, + 'user_services': userServices + }) + logger.debug('Report Data Done') + return self.templateAsPDF( + 'uds/reports/stats/authenticator_stats.html', + dct={ + 'data': stats + }, + header=ugettext('Users usage list'), + water=ugettext('UDS Report of users usage') + ) diff --git a/server/src/uds/reports/stats/base.py b/server/src/uds/reports/stats/base.py index 4422573e4..44f33e164 100644 --- a/server/src/uds/reports/stats/base.py +++ b/server/src/uds/reports/stats/base.py @@ -34,6 +34,7 @@ import typing from django.utils.translation import ugettext_noop as _ from uds.core import reports +from ..auto import ReportAuto class StatsReport(reports.Report): group = _('Statistics') # So we can make submenus with reports @@ -41,3 +42,6 @@ class StatsReport(reports.Report): def generate(self) -> typing.Union[str, bytes]: raise NotImplementedError('StatsReport generate invoked and not implemented') + +class StatsReportAuto(ReportAuto, StatsReport): + pass diff --git a/server/src/uds/templates/uds/reports/stats/authenticator_stats.html b/server/src/uds/templates/uds/reports/stats/authenticator_stats.html new file mode 100644 index 000000000..9c41258f3 --- /dev/null +++ b/server/src/uds/templates/uds/reports/stats/authenticator_stats.html @@ -0,0 +1,40 @@ +{% load l10n i18n %} + + + {% trans 'Users usage list' %} + + + + + + + + + + + + + + + + + {% for d in data %} + + + {% if d.users == None %} + + {% else %} + + + + + {% endif %} + + {% endfor %} + +
{% trans 'Date' %}{% trans 'Users' %}{% trans 'Services' %}{% trans 'User working' %}
{{ d.date }}{{ d.date }}{{ d.users }}{{ d.services }}{{ d.user_services }}
+ + + \ No newline at end of file