burmatovan/openuds
1
0
Fork 0
forked from shaba/openuds
Code Issues Pull Requests Projects Releases Wiki Activity

fixed stats reports & related

Browse Source
This commit is contained in:
Adolfo Gómez García 2019-09-04 11:25:36 +02:00
parent 6e7b5229d1
commit f464d78f99
13 changed files with 108 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/uds/core/reports/graphs.py
View File

@ -45,7 +45,7 @@ import numpy as np
logger = logging.getLogger(__name__)
def barChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO):
def barChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO) -> None:
d = data['x']
ind = np.arange(len(d))
ys = data['y']
@ -77,7 +77,7 @@ def barChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typin
fig.savefig(output, format='png', transparent=True)
def lineChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO):
def lineChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO) -> None:
x = data['x']
y = data['y']
@ -107,7 +107,7 @@ def lineChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typi
fig.savefig(output, format='png', transparent=True)
def surfaceChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO):
def surfaceChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO) -> None:
x = data['x']
y = data['y']
z = data['z']

14
server/src/uds/core/reports/report.py
View File

@ -48,13 +48,13 @@ logger = logging.getLogger(__name__)
class Report(UserInterface):
mime_type: str = 'application/pdf' # Report returns pdfs by default, but could be anything else
name: str = _('Base Report') # Report name
description: str = _('Base report') # Report description
filename: str = 'file.pdf' # Filename that will be returned as 'hint' on rest report request
group: str = '' # So we can "group" reports by kind?
encoded: bool = True # If the report is mean to be encoded (binary reports as PDFs == True, text reports must be False so utf-8 is correctly threated
uuid: str = ''
mime_type: typing.ClassVar[str] = 'application/pdf' # Report returns pdfs by default, but could be anything else
name: typing.ClassVar[str] = _('Base Report') # Report name
description: typing.ClassVar[str] = _('Base report') # Report description
filename: typing.ClassVar[str] = 'file.pdf' # Filename that will be returned as 'hint' on rest report request
group: typing.ClassVar[str] = '' # So we can "group" reports by kind?
encoded: typing.ClassVar[bool] = True # If the report is mean to be encoded (binary reports as PDFs == True, text reports must be False so utf-8 is correctly threated
uuid: typing.ClassVar[str] = ''
@classmethod
def translated_name(cls):

22
server/src/uds/reports/__init__.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2012 Virtual Cable S.L.
# Copyright (c) 2012-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -39,18 +39,12 @@ The registration of modules is done locating subclases of :py:class:`uds.core.au
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
from __future__ import unicode_literals
import six
import logging
logger = logging.getLogger(__name__)
__updated__ = '2018-02-07'
availableReports = []
# noinspection PyTypeChecker
def __init__():
"""
@ -62,25 +56,23 @@ def __init__():
from uds.core import reports
def addReportCls(cls):
logger.debug('Adding report {}'.format(cls))
logger.debug('Adding report %s', cls)
availableReports.append(cls)
def recursiveAdd(p):
if p.uuid is not None:
addReportCls(p)
def recursiveAdd(reportClass):
if reportClass.uuid:
addReportCls(reportClass)
else:
logger.debug('Report class {} not added because it lacks of uuid (it is probably a base class)'.format(p))
logger.debug('Report class %s not added because it lacks of uuid (it is probably a base class)', reportClass)
for c in p.__subclasses__():
for c in reportClass.__subclasses__():
recursiveAdd(c)
# Dinamycally import children of this package. The __init__.py files must import classes
pkgpath = os.path.dirname(sys.modules[__name__].__file__)
# TODO: Make this work with python3 also!!! (look for alternative, we have time...)
for _, name, _ in pkgutil.iter_modules([pkgpath]):
__import__(name, globals(), locals(), [], 1)
recursiveAdd(reports.Report)
__init__()

5
server/src/uds/reports/lists/__init__.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -31,4 +31,5 @@
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
from .users import ListReportUsers, ListReportsUsersCSV
# Make reports visible to autoloader
from . import users

7
server/src/uds/reports/lists/base.py
View File

@ -30,11 +30,14 @@
"""
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
import typing
from django.utils.translation import ugettext_noop as _
from uds.core import reports
class ListReport(reports.Report):
def generate(self):
group = _('Lists') # So we can make submenus with reports
def generate(self) -> typing.Union[str, bytes]:
raise NotImplementedError('ListReport generate invoked and not implemented')
group = _('Lists') # So we can make submenus with reports

10
server/src/uds/reports/stats/__init__.py
View File

@ -31,16 +31,10 @@
@author: Adolfo Gómez, dkmaster at dkmon dot com
"""
# We just need to import. Report loader will look for Report subclasses
# Make reports visible to autoloader
# from . import usage
from uds.reports.stats import user_access
from . import user_access
from . import pools_performance
from . import pools_usage_day
from . import usage_by_pool
from . import pool_usage_summary
# from .user_access import StatsReportLogin, StatsReportLoginCSV
# from .pools_performance import PoolPerformanceReport
# from .pools_usage_day import CountersPoolAssigned
# from .usage_by_pool import UsageByPool
# from .pool_usage_summary import UsageSummaryByPool

14
server/src/uds/reports/stats/base.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -30,16 +30,14 @@
"""
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
from __future__ import unicode_literals
import typing
from django.utils.translation import ugettext_noop as _
from uds.core import reports
__updated__ = '2018-02-07'
class StatsReport(reports.Report):
"""
Base report for statistics reports
"""
group = _('Statistics') # So we can make submenus with reports
def generate(self) -> typing.Union[str, bytes]:
raise NotImplementedError('StatsReport generate invoked and not implemented')

45
server/src/uds/reports/stats/pool_usage_summary.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -30,27 +30,22 @@
"""
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
from __future__ import unicode_literals
import io
import csv
import datetime
import logging
import typing
from django.utils.translation import ugettext, ugettext_lazy as _
from uds.core.ui import gui
from uds.core.util.stats import events
import io
import csv
from uds.models import ServicePool
from .base import StatsReport
from uds.models import ServicePool
import datetime
import logging
logger = logging.getLogger(__name__)
__updated__ = '2018-04-25'
class UsageSummaryByPool(StatsReport):
filename = 'pools_usage.pdf'
@ -92,15 +87,15 @@ class UsageSummaryByPool(StatsReport):
]
self.pool.setValues(vals)
def getPoolData(self, pool):
def getPoolData(self, pool) -> typing.Tuple[typing.List[typing.Dict[str, typing.Any]], str]:
start = self.startDate.stamp()
end = self.endDate.stamp()
logger.debug(self.pool.value)
items = events.statsManager().getEvents(events.OT_DEPLOYED, (events.ET_LOGIN, events.ET_LOGOUT), owner_id=pool.id, since=start, to=end).order_by('stamp')
logins = {}
users = {}
logins: typing.Dict[str, int] = {}
users: typing.Dict[str, typing.Dict] = {}
for i in items:
# if '\\' in i.fld1:
# continue
@ -113,7 +108,7 @@ class UsageSummaryByPool(StatsReport):
del logins[username]
total = i.stamp - stamp
if username not in users:
users[username] = { 'sessions': 0, 'time': 0 }
users[username] = {'sessions': 0, 'time': 0}
users[username]['sessions'] += 1
users[username]['time'] += total
# data.append({
@ -123,18 +118,16 @@ class UsageSummaryByPool(StatsReport):
# })
# Extract different number of users
data = []
for k, v in users.items():
data.append({
'user': k,
'sessions': v['sessions'],
'hours': '{:.2f}'.format(float(v['time']) / 3600),
'average': '{:.2f}'.format(float(v['time']) / 3600 / v['sessions'])
})
data = [{
'user': k,
'sessions': v['sessions'],
'hours': '{:.2f}'.format(float(v['time']) / 3600),
'average': '{:.2f}'.format(float(v['time']) / 3600 / v['sessions'])
} for k, v in users.items()]
return data, pool.name
def getData(self):
def getData(self) -> typing.Tuple[typing.List[typing.Dict[str, typing.Any]], str]:
return self.getPoolData(ServicePool.objects.get(uuid=self.pool.value))
def generate(self):
@ -168,7 +161,7 @@ class UsageSummaryByPoolCSV(UsageSummaryByPool):
output = io.StringIO()
writer = csv.writer(output)
reportData, poolName = self.getData()
reportData = self.getData()[0]
writer.writerow([ugettext('User'), ugettext('Sessions'), ugettext('Hours'), ugettext('Average')])

41
server/src/uds/reports/stats/pools_performance.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015-2018 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -34,6 +34,7 @@ import io
import csv
import datetime
import logging
import typing
from django.utils.translation import ugettext, ugettext_lazy as _
from django.db.models import Count
@ -41,18 +42,14 @@ import django.template.defaultfilters as filters
from uds.core.ui import gui
from uds.core.util.stats import events
from uds.core.util import tools
from uds.core.reports import graphs
from uds.models import ServicePool
from .base import StatsReport
from uds.core.util import tools
from uds.models import ServicePool
logger = logging.getLogger(__name__)
__updated__ = '2018-04-24'
# several constants as Width height, margins, ..
WIDTH, HEIGHT, DPI = 19.2, 10.8, 100
SIZE = (WIDTH, HEIGHT, DPI)
@ -108,10 +105,10 @@ class PoolPerformanceReport(StatsReport):
]
self.pools.setValues(vals)
def getPools(self):
def getPools(self) -> typing.List[typing.Tuple[str, str]]:
return [(v.id, v.name) for v in ServicePool.objects.filter(uuid__in=self.pools.value)]
def getRangeData(self):
def getRangeData(self) -> typing.Tuple[str, typing.List, typing.List]: # pylint: disable=too-many-locals
start = self.startDate.stamp()
end = self.endDate.stamp()
@ -126,10 +123,10 @@ class PoolPerformanceReport(StatsReport):
pools = self.getPools()
if len(pools) == 0:
if pools:
raise Exception(_('Select at least a service pool for the report'))
logger.debug('Pools: {}'.format(pools))
logger.debug('Pools: %s', pools)
# x axis label format
if end - start > 3600 * 24 * 2:
@ -138,7 +135,7 @@ class PoolPerformanceReport(StatsReport):
xLabelFormat = 'SHORT_DATETIME_FORMAT'
# Generate samplings interval
samplingIntervals = []
samplingIntervals: typing.List[typing.Tuple[int, int]] = []
prevVal = None
for val in range(start, end, int((end - start) / (samplingPoints + 1))):
if prevVal is None:
@ -199,12 +196,10 @@ class PoolPerformanceReport(StatsReport):
'x': X,
'xtickFnc': lambda l: filters.date(datetime.datetime.fromtimestamp(X[int(l)]), xLabelFormat) if int(l) >= 0 else '',
'xlabel': _('Date'),
'y': [
{
'label': p['name'],
'data': [v[1] for v in p['dataUsers']]
}
for p in poolsData],
'y': [{
'label': p['name'],
'data': [v[1] for v in p['dataUsers']]
} for p in poolsData],
'ylabel': _('Users')
}
@ -216,12 +211,10 @@ class PoolPerformanceReport(StatsReport):
'x': X,
'xtickFnc': lambda l: filters.date(datetime.datetime.fromtimestamp(X[int(l)]), xLabelFormat) if int(l) >= 0 else '',
'xlabel': _('Date'),
'y': [
{
'label': p['name'],
'data': [v[1] for v in p['dataAccesses']]
}
for p in poolsData],
'y': [{
'label': p['name'],
'data': [v[1] for v in p['dataAccesses']]
} for p in poolsData],
'ylabel': _('Accesses')
}

30
server/src/uds/reports/stats/pools_usage_day.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -30,25 +30,25 @@
"""
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
from django.utils.translation import ugettext, ugettext_lazy as _
from uds.core.ui import gui
from uds.core.util.stats import counters
import csv
import io
import datetime
import logging
import typing
from django.utils.translation import ugettext, ugettext_lazy as _
from uds.core.ui import gui
from uds.core.util.stats import counters
from uds.core.reports import graphs
from uds.models import ServicePool
from .base import StatsReport
from uds.models import ServicePool
from uds.core.reports import graphs
logger = logging.getLogger(__name__)
__updated__ = '2018-04-25'
# several constants as Width height, margins, ..
WIDTH, HEIGHT, DPI = 19.2, 10.8, 100
SIZE = (WIDTH, HEIGHT, DPI)
@ -85,19 +85,19 @@ class CountersPoolAssigned(StatsReport):
]
self.pools.setValues(vals)
def getData(self):
def getData(self) -> typing.List[typing.Dict[str, typing.Any]]:
# Generate the sampling intervals and get dataUsers from db
start = self.startDate.date()
end = self.startDate.date() + datetime.timedelta(days=1)
data = []
pool = None
pool: ServicePool
for poolUuid in self.pools.value:
try:
pool = ServicePool.objects.get(uuid=poolUuid)
except Exception:
pass # Ignore pool
continue
hours = {}
for i in range(24):
@ -111,7 +111,7 @@ class CountersPoolAssigned(StatsReport):
data.append({'uuid':pool.uuid, 'name': pool.name, 'hours': hours})
logger.debug('data: {}'.format(data))
logger.debug('data: %s', data)
return data
@ -124,7 +124,7 @@ class CountersPoolAssigned(StatsReport):
d = {
'title': _('Services by hour'),
'x': X,
'xtickFnc': lambda l: '{:02d}'.format(l),
'xtickFnc': lambda xx: '{:02d}'.format(xx), # pylint: disable=unnecessary-lambda
'xlabel': _('Hour'),
'y': [
{

15
server/src/uds/reports/stats/usage.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -30,18 +30,17 @@
"""
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
from __future__ import unicode_literals
from django.utils.translation import ugettext, ugettext_noop as _
""" from django.utils.translation import ugettext_noop as _
from .base import StatsReport
__updated__ = '2017-05-03'
class StatsReportUsage(object): # Disabled from being used
# class StatsReportUsage(StatsReport):
class StatsReportUsage(StatsReport): # Disabled from being used
name = _('Usage stats') # Report name
description = _('Statistics of platform use') # Report description
uuid = '9ae54172-ed48-11e4-b8e1-10feed05884b'
def generate(self):
return ''
"""

26
server/src/uds/reports/stats/usage_by_pool.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -30,27 +30,23 @@
"""
.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com
"""
from __future__ import unicode_literals
import io
import csv
import datetime
import logging
import typing
from django.utils.translation import ugettext, ugettext_lazy as _
from uds.core.ui import gui
from uds.core.util.stats import events
import six
import csv
from uds.models import ServicePool
from .base import StatsReport
from uds.models import ServicePool
import datetime
import logging
logger = logging.getLogger(__name__)
__updated__ = '2018-04-25'
class UsageByPool(StatsReport):
filename = 'pools_usage.pdf'
@ -92,7 +88,7 @@ class UsageByPool(StatsReport):
]
self.pool.setValues(vals)
def getData(self):
def getData(self) -> typing.Tuple[typing.List[typing.Dict[str, typing.Any]], str]:
# Generate the sampling intervals and get dataUsers from db
start = self.startDate.stamp()
end = self.endDate.stamp()
@ -120,7 +116,7 @@ class UsageByPool(StatsReport):
'time': total
})
logger.debug('data: {}'.format(data))
logger.debug('data: %s', data)
return data, pool.name
@ -150,10 +146,10 @@ class UsageByPoolCSV(UsageByPool):
endDate = UsageByPool.endDate
def generate(self):
output = six.StringIO()
output = io.StringIO()
writer = csv.writer(output)
reportData, poolName = self.getData()
reportData = self.getData()[0]
writer.writerow([ugettext('Date'), ugettext('User'), ugettext('Seconds')])

13
server/src/uds/reports/stats/user_access.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (c) 2015-2018 Virtual Cable S.L.
# Copyright (c) 2015-2019 Virtual Cable S.L.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
@ -34,22 +34,21 @@ import csv
import io
import datetime
import logging
import typing
from django.utils.translation import ugettext, ugettext_lazy as _
import django.template.defaultfilters as filters
from uds.core.ui import gui
from uds.core.util.stats import events
from uds.core.util import tools
from uds.core.reports import graphs
from .base import StatsReport
from uds.core.util import tools
logger = logging.getLogger(__name__)
__updated__ = '2018-04-25'
# several constants as Width height
WIDTH, HEIGHT, DPI = 19.2, 10.8, 100
SIZE = (WIDTH, HEIGHT, DPI)
@ -94,7 +93,7 @@ class StatsReportLogin(StatsReport):
def initGui(self):
pass
def getRangeData(self):
def getRangeData(self) -> typing.Tuple[str, typing.List, typing.List]:
start = self.startDate.stamp()
end = self.endDate.stamp()
if self.samplingPoints.num() < 8:
@ -112,7 +111,7 @@ class StatsReportLogin(StatsReport):
else:
xLabelFormat = 'SHORT_DATETIME_FORMAT'
samplingIntervals = []
samplingIntervals: typing.List[typing.Tuple[int, int]] = []
prevVal = None
for val in range(start, end, int((end - start) / (samplingPoints + 1))):
if prevVal is None:
@ -148,7 +147,7 @@ class StatsReportLogin(StatsReport):
dataWeek[s.weekday()] += 1
dataHour[s.hour] += 1
dataWeekHour[s.weekday()][s.hour] += 1
logger.debug('Data: {} {}'.format(s.weekday(), s.hour))
logger.debug('Data: %s %s', s.weekday(), s.hour)
return dataWeek, dataHour, dataWeekHour