Added new report "Authenticators stats"

server/src/uds/core/ui/__init__.py

@@ -5,4 +5,4 @@ This module contains the definition of UserInterface, needed to describe the int
 between an UDS module and the administration interface
 """
 
-from .user_interface import gui, UserInterface
+from .user_interface import gui, UserInterface, UserInterfaceType

server/src/uds/core/ui/user_interface.py

@@ -103,16 +103,17 @@ class gui:
 
     # Helpers
     @staticmethod
-    def convertToChoices(vals: typing.Iterable[str]) -> typing.List[typing.Dict[str, str]]:
+    def convertToChoices(vals: typing.Union[typing.List[str], typing.MutableMapping[str, str]]) -> typing.List[typing.Dict[str, str]]:
         """
         Helper to convert from array of strings to the same dict used in choice,
         multichoice, ..
         The id is set to values in the array (strings), while text is left empty.
         """
-        res = []
+        if isinstance(vals, (list, tuple)):
-        for v in vals:
+            return [{'id': v, 'text': ''} for v in vals]
-            res.append({'id': v, 'text': ''})
+        
-        return res
+        # Dictionary
+        return [{'id': k, 'text': v} for k, v in vals.items()]
 
     @staticmethod
     def convertToList(vals: typing.Iterable[str]) -> typing.List[str]:
@@ -672,6 +673,8 @@ class gui:
 
         def __init__(self, **options):
             super().__init__(**options)
+            if options.get('values') and isinstance(options.get('values'), dict):
+                options['values'] = gui.convertToChoices(options['values'])
             self._data['values'] = options.get('values', [])
             if 'fills' in options:
                 # Save fnc to register as callback
@@ -738,6 +741,8 @@ class gui:
 
         def __init__(self, **options):
             super().__init__(**options)
+            if options.get('values') and isinstance(options.get('values'), dict):
+                options['values'] = gui.convertToChoices(options['values'])
             self._data['values'] = options.get('values', [])
             self._data['rows'] = options.get('rows', -1)
             self._type(gui.InputField.MULTI_CHOICE_TYPE)
@@ -826,7 +831,7 @@ class UserInterfaceType(type):
         return type.__new__(cls, classname, bases, newClassDict)
 
 
-class UserInterface(metaclass=UserInterfaceType):
+class   UserInterface(metaclass=UserInterfaceType):
     """
     This class provides the management for gui descriptions (user forms)
 

server/src/uds/models/stats_counters.py

@@ -130,12 +130,17 @@ class StatsCounters(models.Model):
         if limit:
             filt += ' LIMIT {}'.format(limit)
 
-        fnc = getSqlFnc('MAX' if kwargs.get('use_max', False) else 'AVG')
+        if kwargs.get('use_max', False):
+            fnc = getSqlFnc('MAX') + ('(value)')
+        else:
+            fnc = getSqlFnc('CEIL') + '({}(value))'.format(getSqlFnc('AVG'))
+
+        # fnc = getSqlFnc('MAX' if kwargs.get('use_max', False) else 'AVG')
 
         query = (
-            'SELECT -1 as id,-1 as owner_id,-1 as owner_type,-1 as counter_type, ' + stampValue + '*{}'.format(interval) + ' AS stamp,' +
+            'SELECT -1 as id,-1 as owner_id,-1 as owner_type,-1 as counter_type, ' + stampValue + '*{}'.format(interval) + ' AS stamp, ' +
-            getSqlFnc('CEIL') + '({0}(value)) AS value '
+            '{} AS value '
-            'FROM {1} WHERE {2}'
+            'FROM {} WHERE {}'
         ).format(fnc, StatsCounters._meta.db_table, filt)
 
         logger.debug('Stats query: %s', query)

server/src/uds/reports/__init__.py

@@ -57,12 +57,15 @@ def __init__():
     """
     This imports all packages that are descendant of this package, and, after that,
     """
+    alreadyAdded: typing.Set[str] = set()
+
     def addReportCls(cls: typing.Type[reports.Report]):
         logger.debug('Adding report %s', cls)
         availableReports.append(cls)
 
     def recursiveAdd(reportClass: typing.Type[reports.Report]):
-        if reportClass.uuid:
+        if reportClass.uuid and reportClass.uuid not in alreadyAdded:
+            alreadyAdded.add(reportClass.uuid)
             addReportCls(reportClass)
         else:
             logger.debug('Report class %s not added because it lacks of uuid (it is probably a base class)', reportClass)
@@ -77,8 +80,9 @@ def __init__():
         # __import__(name, globals(), locals(), [], 1)
         importlib.import_module('.' + name, __name__)  # Local import
 
-    importlib.invalidate_caches()
-
     recursiveAdd(reports.Report)
 
+    importlib.invalidate_caches()
+
+
 __init__()

server/src/uds/reports/auto/__init__.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+
+#
+# Copyright (c) 2015-2019 Virtual Cable S.L.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
+"""
+import logging
+import datetime
+import typing
+
+from django.utils.translation import ugettext, ugettext_noop as _
+
+from uds.core.ui import UserInterface, UserInterfaceType, gui
+from uds.core.reports import Report
+from uds import models
+
+from . import fields
+
+logger = logging.getLogger(__name__)
+
+ReportAutoModel = typing.TypeVar('ReportAutoModel', models.Authenticator, models.ServicePool, models.Service, models.Provider)
+
+class ReportAutoType(UserInterfaceType):
+    def __new__(cls, name, bases, attrs) -> 'ReportAutoType':
+        # Add gui for elements...
+        order = 1
+
+        # Check what source
+        if attrs.get('data_source'):
+
+            attrs['source'] = fields.source_field(order, attrs['data_source'], attrs['multiple'])
+            order += 1
+
+            # Check if date must be added
+            if attrs.get('dates') == 'single':
+                attrs['date_start'] = fields.single_date_field(order)
+                order += 1
+
+            if attrs.get('dates') == 'range':
+                attrs['date_start'] = fields.start_date_field(order)
+                order += 1
+                attrs['date_end'] = fields.end_date_field(order)
+                order += 1
+                
+            # Check if data interval should be included
+            if attrs.get('intervals'):
+                attrs['interval'] = fields.intervals_field(order)
+                order += 1
+
+        return UserInterfaceType.__new__(cls, name, bases, attrs)
+
+class ReportAuto(Report, metaclass=ReportAutoType):
+    # Variables that will be overwriten on new class creation
+    source: typing.ClassVar[typing.Union[gui.MultiChoiceField, gui.ChoiceField]]
+    date_start: typing.ClassVar[gui.DateField]
+    date_end: typing.ClassVar[gui.DateField]
+    interval: typing.ClassVar[gui.ChoiceField]
+
+    # Dates can be None, 'single' or 'range' to auto add date fields
+    dates: typing.ClassVar[typing.Optional[str]] = None
+    intervals: bool = False
+    # Valid data_source:
+    # * ServicePool.usage
+    # * ServicePool.assigned
+    # * Authenticator.users
+    # * Authenticator.services
+    # * Authenticator.user_with_services
+    data_source: str = ''
+
+    # If True, will allow selection of multiple "source" elements
+    multiple: bool = False
+
+    def getModel(self) -> typing.Type[ReportAutoModel]:
+        data_source = self.data_source.split('.')[0]
+
+        return typing.cast(ReportAutoModel, {
+            'ServicePool': models.ServicePool,
+            'Authenticator': models.Authenticator,
+            'Service': models.Service,
+            'Provider': models.Provider
+        }[data_source])
+
+    def initGui(self):
+        # Fills datasource 
+        fields.source_field_data(self.getModel(), self.data_source, self.source)
+
+    def getModelItems(self) -> typing.Iterable[ReportAutoModel]:
+        model = self.getModel()
+        
+        filters = [self.source.value] if isinstance(self.source, gui.ChoiceField) else self.source.value
+
+        if '0-0-0-0' in filters:
+            items = model.objects.all()
+        else:
+            items = model.objects.filter(uuid__in=filters)
+
+        return items
+    
+    def getIntervalInHours(self):
+        return {
+            'hour': 1,
+            'day': 24,
+            'week': 24*7,
+            'month': 24*30
+        }[self.interval.value]

server/src/uds/reports/auto/fields.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+
+#
+# Copyright (c) 2020 Virtual Cable S.L.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
+"""
+import datetime
+import logging
+import typing
+
+from django.utils.translation import ugettext_noop as _
+
+from uds.core.ui import gui
+from uds import models
+
+logger = logging.getLogger(__name__)
+
+def start_date_field(order:int) -> gui.DateField:
+    return gui.DateField(
+        order=order,
+        label=_('Starting date'),
+        tooltip=_('Starting date for report'),
+        defvalue=datetime.date.min,
+        required=True
+    )
+
+def single_date_field(order: int) -> gui.DateField:
+    return  gui.DateField(
+        order=order,
+        label=_('Date'),
+        tooltip=_('Date for report'),
+        defvalue=datetime.date.today(),
+        required=True
+    )
+
+
+def end_date_field(order: int) -> gui.DateField:
+    return gui.DateField(
+        order=order,
+        label=_('Ending date'),
+        tooltip=_('ending date for report'),
+        defvalue=datetime.date.max,
+        required=True
+    )
+
+def intervals_field(order: int) -> gui.ChoiceField:
+    return gui.ChoiceField(
+        label=_('Report data interval'),
+        order=order,
+        values={
+            'hour': _('Hourly'),
+            'day': _('Daily'),
+            'week': _('Weekly'),
+            'month': _('Monthly')
+        },
+        tooltip=_('Interval for report data'),
+        required=True,
+        defvalue='day'
+    )
+
+def source_field(order: int, data_source: str, multiple: bool) -> typing.Union[gui.ChoiceField, gui.MultiChoiceField, None]:
+    if not data_source:
+        return None
+
+    data_source = data_source.split('.')[0]
+    logger.debug('SOURCE: %s', data_source)
+
+    fieldType: typing.Type = gui.ChoiceField if not multiple else gui.MultiChoiceField
+
+    labels: typing.Any = {
+        'ServicePool': (_('Service pool'), _('Service pool for report')),
+        'Authenticator': (_('Authenticator'), _('Authenticator for report')),
+        'Service': (_('Service'), _('Service for report')),
+        'Provider': (_('Service provider'), _('Service provider for report')),
+    }.get(data_source)
+
+    logger.debug('Labels: %s, %s', labels, fieldType)
+
+    return fieldType(
+        label=labels[0],
+        order=order,
+        tooltip=labels[1],
+        required=True
+    )
+
+def source_field_data(model: typing.Any, data_source: str, field: typing.Union[gui.ChoiceField, gui.MultiChoiceField]) -> None:
+
+    dataList: typing.List[gui.ChoiceType] = [{'id': x.uuid, 'text': x.name} for x in model.objects.all().order_by('name')]
+
+    if isinstance(field, gui.MultiChoiceField):
+        dataList.insert(0, {'id': '0-0-0-0', 'text': _('All')})
+
+    field.setValues(dataList)

server/src/uds/reports/stats/__init__.py

@@ -40,3 +40,4 @@ from . import usage_by_pool
 from . import pool_users_summary
 from . import pools_usage_summary
 
+from . import auth_stats

server/src/uds/reports/stats/auth_stats.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+#
+# Copyright (c) 2015-2019 Virtual Cable S.L.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
+"""
+import io
+import csv
+import datetime
+import logging
+import typing
+
+from django.utils.translation import ugettext, ugettext_lazy as _
+
+from uds.core.ui import gui
+from uds.core.util.stats import counters
+
+from .base import StatsReportAuto
+
+logger = logging.getLogger(__name__)
+
+MAX_ELEMENTS = 10000
+
+class AuthenticatorsStats(StatsReportAuto):
+    dates = 'range'
+    intervals = True
+    data_source = 'Authenticator'
+    multiple = True
+
+    filename = 'auths_stats.pdf'
+    name = _('Statistics by authenticator')  # Report name
+    description = _('Generates a report with the statistics of an authenticator for a desired period')  # Report description
+    uuid = 'a5a43bc0-d543-11ea-af8f-af01fa65994e'
+
+    def generate(self) -> typing.Any:
+        since = self.date_start.date()
+        to = self.date_end.date()
+        interval = self.getIntervalInHours() * 3600
+
+        stats = []
+        for a in self.getModelItems():
+            # Will show a.name on every change...
+            stats.append({'date': a.name, 'users': None})
+
+            services = 0
+            userServices = 0
+            servicesCounterIter = iter(counters.getCounters(a, counters.CT_AUTH_SERVICES, since=since, interval=interval, limit=MAX_ELEMENTS, use_max=True))
+            usersWithServicesCounterIter = iter(counters.getCounters(a, counters.CT_AUTH_USERS_WITH_SERVICES, since=since, interval=interval, limit=MAX_ELEMENTS, use_max=True))
+            for userCounter in counters.getCounters(a, counters.CT_AUTH_USERS, since=since, interval=interval, limit=MAX_ELEMENTS, use_max=True):
+                try:
+                    while True:
+                        servicesCounter = next(servicesCounterIter)
+                        if servicesCounter[0] >= userCounter[0]:
+                            break
+                    if userCounter[0] == servicesCounter[0]:
+                        services = servicesCounter[1]
+                except StopIteration:
+                    pass
+
+                try:
+                    while True:
+                        uservicesCounter = next(usersWithServicesCounterIter)
+                        if uservicesCounter[0] >= userCounter[0]:
+                            break
+                    if userCounter[0] == uservicesCounter[0]:
+                        userServices = uservicesCounter[1]
+                except StopIteration:
+                    pass
+
+                stats.append({
+                    'date': userCounter[0],
+                    'users': userCounter[1] or 0,
+                    'services': services,
+                    'user_services': userServices
+                })
+        logger.debug('Report Data Done')
+        return self.templateAsPDF(
+            'uds/reports/stats/authenticator_stats.html',
+            dct={
+                'data': stats
+            },
+            header=ugettext('Users usage list'),
+            water=ugettext('UDS Report of users usage')
+        )

server/src/uds/reports/stats/base.py

@@ -34,6 +34,7 @@ import typing
 
 from django.utils.translation import ugettext_noop as _
 from uds.core import reports
+from ..auto import ReportAuto
 
 class StatsReport(reports.Report):
     group = _('Statistics')  # So we can make submenus with reports
@@ -41,3 +42,6 @@ class StatsReport(reports.Report):
     def generate(self) -> typing.Union[str, bytes]:
         raise NotImplementedError('StatsReport generate invoked and not implemented')
 
+
+class StatsReportAuto(ReportAuto, StatsReport):
+    pass

server/src/uds/templates/uds/reports/stats/authenticator_stats.html

@@ -0,0 +1,40 @@
+{% load l10n i18n %}
+<html lang="en">
+    <head>
+        <title>{% trans 'Users usage list' %}</title>
+        <meta name="author" content="UDS">
+        <meta name="description" content="{% trans 'Statistics by Authenticator' %}">
+        <meta name="keywords" content="uds,report,usage,users,list">
+        <meta name="generator" content="UDS Reporting">
+    </head>
+    <body>
+        <table style="width: 100%; font-size: 0.8em;">
+            <thead>
+                <tr>
+                    <th style="width: 20%">{% trans 'Date' %}</th>
+                    <th style="width: 15%">{% trans 'Users' %}</th>
+                    <th style="width: 25%">{% trans 'Services' %}</th>
+                    <th style="width: 20%">{% trans 'User working' %}</th>
+                </tr>
+            </thead>
+            <tbody>
+            {% for d in data %}
+                
+                <tr>
+                    {% if d.users == None %}
+                    <td colspan="4" style="text-align: center; font-weight: bold; font-size: large;">{{ d.date }}</td>
+                    {% else %}
+                    <td>{{ d.date }}</td>
+                    <td>{{ d.users }}</td>
+                    <td>{{ d.services }}</td>
+                    <td>{{ d.user_services }}</td>
+                    {% endif %}
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+        <!-- <p style="page-break-before: always">
+            This is a new page
+        </p> -->
+    </body>
+</html>