From 2f67eacfb61b906e24a6c041b3db8b187fde3bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20G=C3=B3mez=20Garc=C3=ADa?= Date: Thu, 12 Nov 2020 13:55:09 +0100 Subject: [PATCH] * Removing "encoders" module (nonsense and sometimes confusing. Helped on python 2.7, but now... :) * Revised reports to improve type checking --- server/src/uds/core/jobs/delayed_task.py | 7 +- .../src/uds/core/jobs/delayed_task_runner.py | 24 ++-- server/src/uds/core/jobs/job.py | 19 ++- server/src/uds/core/managers/crypto.py | 96 ++++++++----- server/src/uds/core/managers/stats.py | 7 +- server/src/uds/core/module.py | 10 +- server/src/uds/core/reports/__init__.py | 2 +- server/src/uds/core/reports/graphs.py | 29 ++-- server/src/uds/core/reports/report.py | 12 +- server/src/uds/core/ui/user_interface.py | 135 ++++++++++++------ .../src/uds/core/util/middleware/__init__.py | 7 +- server/src/uds/models/__init__.py | 2 +- server/src/uds/models/service_pool.py | 2 + server/src/uds/reports/__init__.py | 10 +- server/src/uds/reports/lists/__init__.py | 3 +- server/src/uds/reports/lists/base.py | 2 +- server/src/uds/reports/lists/users.py | 8 +- server/src/uds/reports/stats/__init__.py | 3 +- server/src/uds/reports/stats/auth_stats.py | 3 +- server/src/uds/reports/stats/base.py | 2 +- .../uds/reports/stats/pool_users_summary.py | 14 +- .../uds/reports/stats/pools_performance.py | 108 +++++++++----- .../src/uds/reports/stats/pools_usage_day.py | 41 +++--- .../uds/reports/stats/pools_usage_summary.py | 60 ++++++-- server/src/uds/reports/stats/usage.py | 3 +- server/src/uds/reports/stats/usage_by_pool.py | 61 +++++--- server/src/uds/reports/stats/user_access.py | 95 +++++++----- 27 files changed, 479 insertions(+), 286 deletions(-) diff --git a/server/src/uds/core/jobs/delayed_task.py b/server/src/uds/core/jobs/delayed_task.py index ac1332f88..6733998d1 100644 --- a/server/src/uds/core/jobs/delayed_task.py +++ b/server/src/uds/core/jobs/delayed_task.py @@ -32,7 +32,7 @@ """ import logging -from uds.core.environment import Environmentable +from uds.core.environment import Environmentable, Environment logger = logging.getLogger(__name__) @@ -47,8 +47,7 @@ class DelayedTask(Environmentable): """ Remember to invoke parent init in derived clases using super(myClass,self).__init__() to let this initialize its own variables """ - super().__init__(None) - + super().__init__(Environment('DelayedTask')) def execute(self) -> None: """ @@ -59,7 +58,6 @@ class DelayedTask(Environmentable): except Exception as e: logger.error('Job %s raised an exception: %s', self.__class__, e) - def run(self) -> None: """ Run method, executes your code. Override this on your classes @@ -67,7 +65,6 @@ class DelayedTask(Environmentable): logging.error("Base run of job called for class") raise NotImplementedError - def register(self, suggestedTime: int, tag: str = '', check: bool = True) -> None: """ Utility method that allows to register a Delayedtask diff --git a/server/src/uds/core/jobs/delayed_task_runner.py b/server/src/uds/core/jobs/delayed_task_runner.py index dba73b953..615711b18 100644 --- a/server/src/uds/core/jobs/delayed_task_runner.py +++ b/server/src/uds/core/jobs/delayed_task_runner.py @@ -28,13 +28,14 @@ """ @author: Adolfo Gómez, dkmaster at dkmon dot com """ -import threading import time -import logging +import codecs import pickle -import typing +import threading from socket import gethostname from datetime import timedelta +import logging +import typing from django.db import connections from django.db import transaction @@ -43,7 +44,6 @@ from django.db.models import Q from uds.models import DelayedTask as DBDelayedTask from uds.models import getSqlDatetime from uds.core.environment import Environment -from uds.core.util import encoders from .delayed_task import DelayedTask @@ -56,7 +56,7 @@ class DelayedTaskThread(threading.Thread): """ _taskInstance: DelayedTask - def __init__(self, taskInstance: DelayedTask): + def __init__(self, taskInstance: DelayedTask) -> None: super().__init__() self._taskInstance = taskInstance @@ -73,11 +73,11 @@ class DelayedTaskRunner: """ Delayed task runner class """ - # How often tasks r checked + # How often tasks are checked granularity: int = 2 # to keep singleton DelayedTaskRunner - _runner: typing.Optional['DelayedTaskRunner'] = None + _runner: typing.ClassVar[typing.Optional['DelayedTaskRunner']] = None _hostname: str _keepRunning: bool @@ -113,16 +113,16 @@ class DelayedTaskRunner: # Throws exception if no delayed task is avilable task = DBDelayedTask.objects.select_for_update().filter(filt).order_by('execution_time')[0] # @UndefinedVariable if task.insert_date > now + timedelta(seconds=30): - logger.warning('EXecuted %s due to insert_date being in the future!', task.type) - taskInstanceDump = typing.cast(bytes, encoders.decode(task.instance, 'base64')) + logger.warning('Executed %s due to insert_date being in the future!', task.type) + taskInstanceDump = codecs.decode(task.instance.encode(), 'base64') task.delete() taskInstance = pickle.loads(taskInstanceDump) except IndexError: return # No problem, there is no waiting delayed task except Exception: # Transaction have been rolled back using the "with atomic", so here just return - # Note that is taskInstance can't be loaded, this task will not be retried - logger.exception('Executing one task') + # Note that is taskInstance can't be loaded, this task will not be run + logger.exception('Obtainint one task for execution') return if taskInstance: @@ -134,7 +134,7 @@ class DelayedTaskRunner: now = getSqlDatetime() exec_time = now + timedelta(seconds=delay) cls = instance.__class__ - instanceDump = encoders.encodeAsStr(pickle.dumps(instance), 'base64') + instanceDump = codecs.encode(pickle.dumps(instance), 'base64').decode() typeName = str(cls.__module__ + '.' + cls.__name__) logger.debug('Inserting delayed task %s with %s bytes (%s)', typeName, len(instanceDump), exec_time) diff --git a/server/src/uds/core/jobs/job.py b/server/src/uds/core/jobs/job.py index b068dd597..b1a40c34a 100644 --- a/server/src/uds/core/jobs/job.py +++ b/server/src/uds/core/jobs/job.py @@ -41,8 +41,12 @@ logger = logging.getLogger(__name__) class Job(Environmentable): # Default frecuency, once a day. Remenber that precision will be based on "granurality" of Scheduler # If a job is used for delayed execution, this attribute is in fact ignored - frecuency: int = 24 * 3600 + 3 # Defaults to a big one, and i know frecuency is written as frequency, but this is an "historical mistake" :) - frecuency_cfg: typing.Optional[Config.Value] = None # If we use a configuration variable from DB, we need to update the frecuency asap, but not before app is ready + frecuency: int = ( + 24 * 3600 + 3 + ) # Defaults to a big one, and i know frecuency is written as frequency, but this is an "historical mistake" :) + frecuency_cfg: typing.Optional[ + Config.Value + ] = None # If we use a configuration variable from DB, we need to update the frecuency asap, but not before app is ready friendly_name = 'Unknown' @classmethod @@ -53,9 +57,16 @@ class Job(Environmentable): if cls.frecuency_cfg: try: cls.frecuency = cls.frecuency_cfg.getInt(force=True) - logger.debug('Setting frequency from DB setting for %s to %s', cls, cls.frecuency) + logger.debug( + 'Setting frequency from DB setting for %s to %s', cls, cls.frecuency + ) except Exception as e: - logger.error('Error setting default frequency for %s ()%s. Got default value of %s', cls, e, cls.frecuency) + logger.error( + 'Error setting default frequency for %s ()%s. Got default value of %s', + cls, + e, + cls.frecuency, + ) def execute(self) -> None: try: diff --git a/server/src/uds/core/managers/crypto.py b/server/src/uds/core/managers/crypto.py index 79f67fefd..e216fddcb 100644 --- a/server/src/uds/core/managers/crypto.py +++ b/server/src/uds/core/managers/crypto.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -33,12 +32,14 @@ import hashlib import array import uuid +import codecs import struct import random import string import logging import typing + from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -51,19 +52,20 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes # cryptography libraries. Keep here for backwards compat with # 1.x 2.x encriptions methods from Crypto.PublicKey import RSA -from Crypto.Random import atfork # type: ignore +from Crypto.Random import atfork # type: ignore from django.conf import settings -from uds.core.util import encoders - logger = logging.getLogger(__name__) + class CryptoManager: instance = None def __init__(self): - self._rsa = serialization.load_pem_private_key(settings.RSA_KEY.encode(), password=None, backend=default_backend()) + self._rsa = serialization.load_pem_private_key( + settings.RSA_KEY.encode(), password=None, backend=default_backend() + ) self._oldRsa = RSA.importKey(settings.RSA_KEY) self._namespace = uuid.UUID('627a37a5-e8db-431a-b783-73f7d20b4934') self._counter = 0 @@ -91,47 +93,41 @@ class CryptoManager: CryptoManager.instance = CryptoManager() return CryptoManager.instance - def encrypt(self, value: typing.Union[str, bytes]) -> str: - if isinstance(value, str): - value = value.encode('utf-8') - - return encoders.encodeAsStr( + def encrypt(self, value: str) -> str: + return codecs.encode( self._rsa.public_key().encrypt( - value, + value.encode(), padding.OAEP( mgf=padding.MGF1(algorithm=hashes.SHA256()), algorithm=hashes.SHA256(), - label=None - ) + label=None, + ), ), - 'base64' - ) + 'base64', + ).decode() # atfork() # return typing.cast(str, encoders.encode((self._rsa.encrypt(value, b'')[0]), 'base64', asText=True)) - - def decrypt(self, value: typing.Union[str, bytes]) -> str: - if isinstance(value, str): - value = value.encode('utf-8') - - data: bytes = typing.cast(bytes, encoders.decode(value, 'base64')) - decrypted: bytes + def decrypt(self, value: str) -> str: + data: bytes = codecs.decode(value.encode(), 'base64') try: # First, try new "cryptografy" decrpypting - decrypted = self._rsa.decrypt( + decrypted: bytes = self._rsa.decrypt( data, padding.OAEP( mgf=padding.MGF1(algorithm=hashes.SHA256()), algorithm=hashes.SHA256(), - label=None - ) + label=None, + ), ) except Exception: # If fails, try old method try: atfork() - decrypted = self._oldRsa.decrypt(encoders.decode(value, 'base64')) + decrypted = self._oldRsa.decrypt( + codecs.decode(value.encode(), 'base64') + ) return decrypted.decode() except Exception: logger.exception('Decripting: %s', value) @@ -142,27 +138,39 @@ class CryptoManager: def AESCrypt(self, text: bytes, key: bytes, base64: bool = False) -> bytes: # First, match key to 16 bytes. If key is over 16, create a new one based on key of 16 bytes length - cipher = Cipher(algorithms.AES(CryptoManager.AESKey(key, 16)), modes.CBC(b'udsinitvectoruds'), backend=default_backend()) - rndStr = self.randomString(16).encode() # Same as block size of CBC (that is 16 here) + cipher = Cipher( + algorithms.AES(CryptoManager.AESKey(key, 16)), + modes.CBC(b'udsinitvectoruds'), + backend=default_backend(), + ) + rndStr = self.randomString( + 16 + ).encode() # Same as block size of CBC (that is 16 here) paddedLength = ((len(text) + 4 + 15) // 16) * 16 - toEncode = struct.pack('>i', len(text)) + text + rndStr[:paddedLength - len(text) - 4] + toEncode = ( + struct.pack('>i', len(text)) + text + rndStr[: paddedLength - len(text) - 4] + ) encryptor = cipher.encryptor() encoded = encryptor.update(toEncode) + encryptor.finalize() - + if base64: - return typing.cast(bytes, encoders.encode(encoded, 'base64')) # Return as binary + return codecs.encode(encoded, 'base64') # Return as binary return encoded def AESDecrypt(self, text: bytes, key: bytes, base64: bool = False) -> bytes: if base64: - text = typing.cast(bytes, encoders.decode(text, 'base64')) + text = codecs.decode(text, 'base64') - cipher = Cipher(algorithms.AES(CryptoManager.AESKey(key, 16)), modes.CBC(b'udsinitvectoruds'), backend=default_backend()) + cipher = Cipher( + algorithms.AES(CryptoManager.AESKey(key, 16)), + modes.CBC(b'udsinitvectoruds'), + backend=default_backend(), + ) decryptor = cipher.decryptor() toDecode = decryptor.update(text) + decryptor.finalize() - return toDecode[4:4 + struct.unpack('>i', toDecode[:4])[0]] + return toDecode[4 : 4 + struct.unpack('>i', toDecode[:4])[0]] def xor(self, s1: typing.Union[str, bytes], s2: typing.Union[str, bytes]) -> bytes: if isinstance(s1, str): @@ -175,7 +183,9 @@ class CryptoManager: # We must return bynary in xor, because result is in fact binary return array.array('B', (s1a[i] ^ s2a[i] for i in range(len(s1a)))).tobytes() - def symCrypt(self, text: typing.Union[str, bytes], key: typing.Union[str, bytes]) -> bytes: + def symCrypt( + self, text: typing.Union[str, bytes], key: typing.Union[str, bytes] + ) -> bytes: if isinstance(text, str): text = text.encode() if isinstance(key, str): @@ -183,7 +193,9 @@ class CryptoManager: return self.AESCrypt(text, key) - def symDecrpyt(self, cryptText: typing.Union[str, bytes], key: typing.Union[str, bytes]) -> str: + def symDecrpyt( + self, cryptText: typing.Union[str, bytes], key: typing.Union[str, bytes] + ) -> str: if isinstance(cryptText, str): cryptText = cryptText.encode() @@ -216,7 +228,11 @@ class CryptoManager: raise Exception('Invalid certificate') def certificateString(self, certificate: str) -> str: - return certificate.replace('-----BEGIN CERTIFICATE-----', '').replace('-----END CERTIFICATE-----', '').replace('\n', '') + return ( + certificate.replace('-----BEGIN CERTIFICATE-----', '') + .replace('-----END CERTIFICATE-----', '') + .replace('\n', '') + ) def hash(self, value: typing.Union[str, bytes]) -> str: if isinstance(value, str): @@ -240,7 +256,9 @@ class CryptoManager: else: obj = '{}'.format(obj) - return str(uuid.uuid5(self._namespace, obj)).lower() # I believe uuid returns a lowercase uuid always, but in case... :) + return str( + uuid.uuid5(self._namespace, obj) + ).lower() # I believe uuid returns a lowercase uuid always, but in case... :) def randomString(self, length: int = 40, digits: bool = True) -> str: base = string.ascii_lowercase + (string.digits if digits else '') diff --git a/server/src/uds/core/managers/stats.py b/server/src/uds/core/managers/stats.py index ee3174082..ee8fef113 100644 --- a/server/src/uds/core/managers/stats.py +++ b/server/src/uds/core/managers/stats.py @@ -145,7 +145,10 @@ class StatsManager: """ self.__doCleanup(StatsCounters) - def getEventFldFor(self, fld: str) -> typing.Optional[str]: + def getEventFldFor(self, fld: str) -> str: + ''' + Get equivalency between "cool names" and field. Will raise "KeyError" if no equivalency + ''' return { 'username': 'fld1', 'platform': 'fld1', @@ -154,7 +157,7 @@ class StatsManager: 'dstip': 'fld3', 'version': 'fld3', 'uniqueid': 'fld4' - }.get(fld, None) + }[fld] # Event stats def addEvent(self, owner_type: int, owner_id: int, eventType: int, **kwargs): diff --git a/server/src/uds/core/module.py b/server/src/uds/core/module.py index 81cd327d0..b093755a8 100644 --- a/server/src/uds/core/module.py +++ b/server/src/uds/core/module.py @@ -30,16 +30,16 @@ """ .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com """ -import typing -import os.path import sys +import os.path +import codecs import logging +import typing from django.utils.translation import ugettext as _ from uds.core.ui import UserInterface -from uds.core.util import encoders from .serializable import Serializable from .environment import Environment, Environmentable @@ -182,8 +182,8 @@ class Module(UserInterface, Environmentable, Serializable): return data @classmethod - def icon64(cls: typing.Type['Module']) -> typing.Union[str]: - return encoders.encodeAsStr(cls.icon(), 'base64') + def icon64(cls: typing.Type['Module']) -> str: + return codecs.encode(cls.icon(), 'base64').decode() @staticmethod def test(env: Environment, data: typing.Dict[str, str]) -> typing.List[typing.Any]: diff --git a/server/src/uds/core/reports/__init__.py b/server/src/uds/core/reports/__init__.py index ef68d462a..93a84d33c 100644 --- a/server/src/uds/core/reports/__init__.py +++ b/server/src/uds/core/reports/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, diff --git a/server/src/uds/core/reports/graphs.py b/server/src/uds/core/reports/graphs.py index 914ae5c08..965569131 100644 --- a/server/src/uds/core/reports/graphs.py +++ b/server/src/uds/core/reports/graphs.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2018-2019 Virtual Cable S.L. +# Copyright (c) 2018-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -37,6 +37,7 @@ import typing from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas from matplotlib.figure import Figure from matplotlib import cm + # This must be imported to allow 3d projections from mpl_toolkits.mplot3d import Axes3D # pylint: disable=unused-import @@ -45,13 +46,15 @@ import numpy as np logger = logging.getLogger(__name__) -def barChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO) -> None: +def barChart( + size: typing.Tuple[float, float, int], data: typing.Dict, output: typing.BinaryIO +) -> None: d = data['x'] ind = np.arange(len(d)) ys = data['y'] width = 0.60 - fig = Figure(figsize=(size[0], size[1]), dpi=size[2]) + fig: typing.Any = Figure(figsize=(size[0], size[1]), dpi=size[2]) # type: ignore FigureCanvas(fig) # Stores canvas on fig.canvas axis = fig.add_subplot(111) @@ -77,11 +80,13 @@ def barChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typin fig.savefig(output, format='png', transparent=True) -def lineChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO) -> None: +def lineChart( + size: typing.Tuple[float, float, int], data: typing.Dict, output: typing.BinaryIO +) -> None: x = data['x'] y = data['y'] - fig = Figure(figsize=(size[0], size[1]), dpi=size[2]) + fig : typing.Any = Figure(figsize=(size[0], size[1]), dpi=size[2]) # type: ignore FigureCanvas(fig) # Stores canvas on fig.canvas axis = fig.add_subplot(111) @@ -107,7 +112,9 @@ def lineChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typi fig.savefig(output, format='png', transparent=True) -def surfaceChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: typing.BinaryIO) -> None: +def surfaceChart( + size: typing.Tuple[float, float, int], data: typing.Dict, output: typing.BinaryIO +) -> None: x = data['x'] y = data['y'] z = data['z'] @@ -123,16 +130,20 @@ def surfaceChart(size: typing.Tuple[int, int, int], data: typing.Dict, output: t logger.debug('Y\': %s', y) logger.debug('Z\': %s', z) - fig = Figure(figsize=(size[0], size[1]), dpi=size[2]) + fig : typing.Any = Figure(figsize=(size[0], size[1]), dpi=size[2]) # type: ignore FigureCanvas(fig) # Stores canvas on fig.canvas axis = fig.add_subplot(111, projection='3d') # axis.grid(color='r', linestyle='dotted', linewidth=0.1, alpha=0.5) if data.get('wireframe', False) is True: - axis.plot_wireframe(x, y, z, rstride=1, cstride=1, cmap=cm.coolwarm) # @UndefinedVariable + axis.plot_wireframe( + x, y, z, rstride=1, cstride=1, cmap=cm.coolwarm + ) # @UndefinedVariable else: - axis.plot_surface(x, y, z, rstride=1, cstride=1, cmap=cm.coolwarm) # @UndefinedVariable + axis.plot_surface( + x, y, z, rstride=1, cstride=1, cmap=cm.coolwarm + ) # @UndefinedVariable axis.set_title(data.get('title', '')) axis.set_xlabel(data['xlabel']) diff --git a/server/src/uds/core/reports/report.py b/server/src/uds/core/reports/report.py index b939458c7..ab9ee8fb0 100644 --- a/server/src/uds/core/reports/report.py +++ b/server/src/uds/core/reports/report.py @@ -30,8 +30,9 @@ """ .. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com """ +import codecs +import datetime import logging -from datetime import datetime import typing from weasyprint import HTML, CSS, default_url_fetcher @@ -40,7 +41,6 @@ from django.utils.translation import ugettext, ugettext_noop as _ from django.template import loader from uds.core.ui import UserInterface, gui -from uds.core.util import encoders from . import stock @@ -116,13 +116,13 @@ class Report(UserInterface): .replace('{page}', _('Page')) .replace('{of}', _('of')) .replace('{water}', water or 'UDS Report') - .replace('{printed}', _('Printed in {now:%Y, %b %d} at {now:%H:%M}').format(now=datetime.now())) + .replace('{printed}', _('Printed in {now:%Y, %b %d} at {now:%H:%M}').format(now=datetime.datetime.now())) ) h = HTML(string=html, url_fetcher=report_fetcher) c = CSS(string=css) - return h.write_pdf(stylesheets=[c]) + return typing.cast(bytes, h.write_pdf(stylesheets=[c])) # Return a new bytes object @staticmethod def templateAsPDF(templateName, dct, header=None, water=None, images=None) -> bytes: @@ -161,7 +161,7 @@ class Report(UserInterface): This can be or can be not overriden """ - def generate(self) -> typing.Union[str, bytes]: + def generate(self) -> bytes: """ Generates the reports @@ -178,7 +178,7 @@ class Report(UserInterface): """ data = self.generate() if self.encoded: - return encoders.encodeAsStr(data, 'base64').replace('\n', '') + return codecs.encode(data, 'base64').decode().replace('\n', '') return typing.cast(str, data) diff --git a/server/src/uds/core/ui/user_interface.py b/server/src/uds/core/ui/user_interface.py index bd8e53a12..f43d00974 100644 --- a/server/src/uds/core/ui/user_interface.py +++ b/server/src/uds/core/ui/user_interface.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -47,6 +46,7 @@ logger = logging.getLogger(__name__) UDSB = b'udsprotect' + class gui: """ This class contains the representations of fields needed by UDS modules and @@ -81,9 +81,13 @@ class gui: can access this form to let users create new instances of this module. """ + # Values dict type ValuesType = typing.Optional[typing.Dict[str, str]] - ValuesDictType = typing.Dict[str, typing.Union[str, bool, typing.List[str], typing.List[typing.Dict[str, str]]]] + ValuesDictType = typing.Dict[ + str, + typing.Union[str, bool, typing.List[str], typing.List[typing.Dict[str, str]]], + ] ChoiceType = typing.Dict[str, str] # : True string value @@ -99,11 +103,16 @@ class gui: DISPLAY_TAB: typing.ClassVar[str] = ugettext_noop('Display') # : Static Callbacks simple registry - callbacks: typing.Dict[str, typing.Callable[[typing.Dict[str, str]], typing.List[typing.Dict[str, str]]]] = {} + callbacks: typing.Dict[ + str, + typing.Callable[[typing.Dict[str, str]], typing.List[typing.Dict[str, str]]], + ] = {} # Helpers @staticmethod - def convertToChoices(vals: typing.Union[typing.List[str], typing.MutableMapping[str, str]]) -> typing.List[typing.Dict[str, str]]: + def convertToChoices( + vals: typing.Union[typing.List[str], typing.MutableMapping[str, str]] + ) -> typing.List[typing.Dict[str, str]]: """ Helper to convert from array of strings to the same dict used in choice, multichoice, .. @@ -111,7 +120,7 @@ class gui: """ if isinstance(vals, (list, tuple)): return [{'id': v, 'text': ''} for v in vals] - + # Dictionary return [{'id': k, 'text': v} for k, v in vals.items()] @@ -141,7 +150,9 @@ class gui: return {'id': str(id_), 'text': str(text)} @staticmethod - def choiceImage(id_: typing.Union[str, int], text: str, img: str) -> typing.Dict[str, str]: + def choiceImage( + id_: typing.Union[str, int], text: str, img: str + ) -> typing.Dict[str, str]: return {'id': str(id_), 'text': str(text), 'img': img} @staticmethod @@ -221,6 +232,7 @@ class gui: so if you use both, the used one will be "value". This is valid for all form fields. """ + TEXT_TYPE: typing.ClassVar[str] = 'text' TEXTBOX_TYPE: typing.ClassVar[str] = 'textbox' NUMERIC_TYPE: typing.ClassVar[str] = 'numeric' @@ -233,21 +245,25 @@ class gui: IMAGECHOICE_TYPE: typing.ClassVar[str] = 'imgchoice' DATE_TYPE: typing.ClassVar[str] = 'date' INFO_TYPE: typing.ClassVar[str] = 'dummy' - - DEFAULT_LENTGH: typing.ClassVar[int] = 64 # : If length of some fields are not especified, this value is used as default + # : If length of some fields are not especified, this value is used as default + DEFAULT_LENTGH: typing.ClassVar[int] = 64 _data: typing.Dict[str, typing.Any] - def __init__(self, **options): + def __init__(self, **options) -> None: defvalue = options.get('defvalue', '') if callable(defvalue): defvalue = defvalue() self._data = { - 'length': options.get('length', gui.InputField.DEFAULT_LENTGH), # Length is not used on some kinds of fields, but present in all anyway + 'length': options.get( + 'length', gui.InputField.DEFAULT_LENTGH + ), # Length is not used on some kinds of fields, but present in all anyway 'required': options.get('required', False), 'label': options.get('label', ''), 'defvalue': str(defvalue), - 'rdonly': options.get('rdonly', False), # This property only affects in "modify" operations + 'rdonly': options.get( + 'rdonly', False + ), # This property only affects in "modify" operations 'order': options.get('order', 0), 'tooltip': options.get('tooltip', ''), 'type': gui.InputField.TEXT_TYPE, @@ -256,7 +272,7 @@ class gui: if 'tab' in options: self._data['tab'] = options.get('tab') - def _type(self, type_: str): + def _type(self, type_: str) -> None: """ Sets the type of this field. @@ -271,7 +287,7 @@ class gui: """ return self._data['type'] == type_ - def isSerializable(self): + def isSerializable(self) -> bool: return True def num(self) -> int: @@ -288,22 +304,26 @@ class gui: returns default value instead. This is mainly used for hidden fields, so we have correctly initialized """ - return self._data['value'] if self._data['value'] is not None else self.defValue + return ( + self._data['value'] + if self._data['value'] is not None + else self.defValue + ) @value.setter - def value(self, value: typing.Any): + def value(self, value: typing.Any) -> None: """ Stores new value (not the default one) """ self._setValue(value) - def _setValue(self, value: typing.Any): + def _setValue(self, value: typing.Any) -> None: """ So we can override value setting at descendants """ self._data['value'] = value - def guiDescription(self): + def guiDescription(self) -> typing.Dict[str, typing.Any]: """ Returns the dictionary with the description of this item. We copy it, cause we need to translate the label and tooltip fields @@ -325,10 +345,10 @@ class gui: return self._data['defvalue'] @defValue.setter - def defValue(self, defValue: typing.Any): + def defValue(self, defValue: typing.Any) -> None: self.setDefValue(defValue) - def setDefValue(self, defValue: typing.Any): + def setDefValue(self, defValue: typing.Any) -> None: """ Sets the default value of the field· @@ -373,7 +393,7 @@ class gui: """ - def __init__(self, **options): + def __init__(self, **options) -> None: super().__init__(**options) self._type(gui.InputField.TEXT_TYPE) multiline = int(options.get('multiline', 0)) @@ -443,7 +463,9 @@ class gui: """ - def processValue(self, valueName: str, options: typing.Dict[str, typing.Any]) -> None: + def processValue( + self, valueName: str, options: typing.Dict[str, typing.Any] + ) -> None: val = options.get(valueName, '') if not val and valueName == 'defvalue': @@ -463,14 +485,29 @@ class gui: super().__init__(**options) self._type(gui.InputField.DATE_TYPE) - def date(self): - try: - return datetime.datetime.strptime(self.value, '%Y-%m-%d').date() # ISO Format - except Exception: - return None + def date(self, min: bool = True) -> datetime.date: + """ + Returns the date tis objecct represents - def stamp(self): - return int(time.mktime(datetime.datetime.strptime(self.value, '%Y-%m-%d').timetuple())) + Args: + min (bool, optional): If true, in case of invalid date will return "min" date, else "max". Defaults to True. + + Returns: + datetime.date: the date that this object holds, or "min" | "max" on error + """ + try: + return datetime.datetime.strptime( + self.value, '%Y-%m-%d' + ).date() # ISO Format + except Exception: + return datetime.date.min if min else datetime.date.max + + def stamp(self) -> int: + return int( + time.mktime( + datetime.datetime.strptime(self.value, '%Y-%m-%d').timetuple() + ) + ) class PasswordField(InputField): """ @@ -692,7 +729,6 @@ class gui: self._data['values'] = values class ImageChoiceField(InputField): - def __init__(self, **options): super().__init__(**options) self._data['values'] = options.get('values', []) @@ -783,7 +819,7 @@ class gui: # : Constant for separating values at "value" method SEPARATOR = '\001' - def __init__(self, **options): + def __init__(self, **options) -> None: super().__init__(**options) self._data['values'] = gui.convertToList(options.get('values', [])) self._type(gui.InputField.EDITABLE_LIST) @@ -800,7 +836,7 @@ class gui: Image field """ - def __init__(self, **options): + def __init__(self, **options) -> None: super().__init__(**options) self._type(gui.InputField.TEXT_TYPE) @@ -809,7 +845,7 @@ class gui: Informational field (no input is done) """ - def __init__(self, **options): + def __init__(self, **options) -> None: super().__init__(**options) self._type(gui.InputField.INFO_TYPE) @@ -819,7 +855,8 @@ class UserInterfaceType(type): Metaclass definition for moving the user interface descriptions to a usable better place """ - def __new__(cls, classname, bases, classDict): # pylint: disable=bad-mcs-classmethod-argument + + def __new__(cls, classname, bases, classDict): newClassDict = {} _gui: typing.Dict[str, gui.InputField] = {} # We will keep a reference to gui elements also at _gui so we can access them easily @@ -831,7 +868,7 @@ class UserInterfaceType(type): return type.__new__(cls, classname, bases, newClassDict) -class UserInterface(metaclass=UserInterfaceType): +class UserInterface(metaclass=UserInterfaceType): """ This class provides the management for gui descriptions (user forms) @@ -845,12 +882,13 @@ class UserInterface(metaclass=UserInterfaceType): _gui: typing.Dict[str, gui.InputField] - - def __init__(self, values: gui.ValuesType = None): + def __init__(self, values: gui.ValuesType = None) -> None: # : If there is an array of elements to initialize, simply try to store values on form fields # Generate a deep copy of inherited Gui, so each User Interface instance has its own "field" set, and do not share the "fielset" with others, what can be really dangerous # Till now, nothing bad happened cause there where being used "serialized", but this do not have to be this way - self._gui = copy.deepcopy(self._gui) # Ensure "gui" is our own instance, deep copied from base + self._gui = copy.deepcopy( + self._gui + ) # Ensure "gui" is our own instance, deep copied from base for key, val in self._gui.items(): # And refresh references to them setattr(self, key, val) @@ -944,11 +982,15 @@ class UserInterface(metaclass=UserInterfaceType): if v.isType(gui.InputField.INFO_TYPE): # logger.debug('Field {} is a dummy field and will not be serialized') continue - if v.isType(gui.InputField.EDITABLE_LIST) or v.isType(gui.InputField.MULTI_CHOICE_TYPE): + if v.isType(gui.InputField.EDITABLE_LIST) or v.isType( + gui.InputField.MULTI_CHOICE_TYPE + ): # logger.debug('Serializing value {0}'.format(v.value)) val = b'\001' + pickle.dumps(v.value, protocol=0) elif v.isType(gui.InfoField.PASSWORD_TYPE): - val = b'\004' + cryptoManager().AESCrypt(v.value.encode('utf8'), UDSB, True) + val = b'\004' + cryptoManager().AESCrypt( + v.value.encode('utf8'), UDSB, True + ) elif v.isType(gui.InputField.NUMERIC_TYPE): val = str(int(v.num())).encode('utf8') elif v.isType(gui.InputField.CHECKBOX_TYPE): @@ -965,7 +1007,7 @@ class UserInterface(metaclass=UserInterfaceType): return typing.cast(bytes, encoders.encode(b'\002'.join(arr), 'zip')) - def unserializeForm(self, values: bytes): + def unserializeForm(self, values: bytes) -> None: """ This method unserializes the values previously obtained using :py:meth:`serializeForm`, and stores @@ -977,7 +1019,10 @@ class UserInterface(metaclass=UserInterfaceType): try: # Set all values to defaults ones for k in self._gui: - if self._gui[k].isType(gui.InputField.HIDDEN_TYPE) and self._gui[k].isSerializable() is False: + if ( + self._gui[k].isType(gui.InputField.HIDDEN_TYPE) + and self._gui[k].isSerializable() is False + ): # logger.debug('Field {0} is not unserializable'.format(k)) continue self._gui[k].value = self._gui[k].defValue @@ -1011,7 +1056,9 @@ class UserInterface(metaclass=UserInterfaceType): # logger.info('Invalid serialization data on {0} {1}'.format(self, values.encode('hex'))) @classmethod - def guiDescription(cls, obj: typing.Optional['UserInterface'] = None) -> typing.List[typing.Dict[str, str]]: + def guiDescription( + cls, obj: typing.Optional['UserInterface'] = None + ) -> typing.List[typing.MutableMapping[str, typing.Any]]: """ This simple method generates the theGui description needed by the administration client, so it can @@ -1027,7 +1074,7 @@ class UserInterface(metaclass=UserInterfaceType): obj.initGui() # We give the "oportunity" to fill necesary theGui data before providing it to client theGui = obj - res = [] + res: typing.List[typing.MutableMapping[str, typing.Any]] = [] # pylint: disable=protected-access,maybe-no-member for key, val in theGui._gui.items(): logger.debug('%s ### %s', key, val) diff --git a/server/src/uds/core/util/middleware/__init__.py b/server/src/uds/core/util/middleware/__init__.py index f418070ab..8b58f6484 100644 --- a/server/src/uds/core/util/middleware/__init__.py +++ b/server/src/uds/core/util/middleware/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2013-2019 Virtual Cable S.L. +# Copyright (c) 2013-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -51,6 +51,11 @@ class XUACompatibleMiddleware: class RedirectMiddleware: + """ + This class is responsible of redirection, if checked, requests to HTTPS. + + Some paths will not be redirected, to avoid problems, but they are advised to use SSL (this is for backwards compat) + """ NO_REDIRECT = [ 'rest', 'pam', diff --git a/server/src/uds/models/__init__.py b/server/src/uds/models/__init__.py index 3f99bda06..fdaf63a04 100644 --- a/server/src/uds/models/__init__.py +++ b/server/src/uds/models/__init__.py @@ -61,7 +61,7 @@ from .group import Group from .service_pool import ServicePool # New name from .meta_pool import MetaPool, MetaPoolMember from .service_pool_group import ServicePoolGroup -from .service_pool_publication import ServicePoolPublication +from .service_pool_publication import ServicePoolPublication, ServicePoolPublicationChangelog from .user_service import UserService from .user_service_property import UserServiceProperty diff --git a/server/src/uds/models/service_pool.py b/server/src/uds/models/service_pool.py index d579f6aea..4a9d926c4 100644 --- a/server/src/uds/models/service_pool.py +++ b/server/src/uds/models/service_pool.py @@ -64,6 +64,7 @@ if typing.TYPE_CHECKING: from uds.models import ( UserService, ServicePoolPublication, + ServicePoolPublicationChangelog, User, Group, Proxy, @@ -157,6 +158,7 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore memberOfMeta: 'models.QuerySet[MetaPoolMember]' userServices: 'models.QuerySet[UserService]' calendarAccess: 'models.QuerySet[CalendarAccess]' + changelog: 'models.QuerySet[ServicePoolPublicationChangelog]' class Meta(UUIDModel.Meta): """ diff --git a/server/src/uds/reports/__init__.py b/server/src/uds/reports/__init__.py index 9846ae17f..3dd996c04 100644 --- a/server/src/uds/reports/__init__.py +++ b/server/src/uds/reports/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2012-2019 Virtual Cable S.L. +# Copyright (c) 2012-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -52,18 +51,17 @@ logger = logging.getLogger(__name__) availableReports: typing.List[typing.Type['reports.Report']] = [] -# noinspection PyTypeChecker -def __init__(): +def __init__() -> None: """ This imports all packages that are descendant of this package, and, after that, """ alreadyAdded: typing.Set[str] = set() - def addReportCls(cls: typing.Type[reports.Report]): + def addReportCls(cls: typing.Type[reports.Report]) -> None: logger.debug('Adding report %s', cls) availableReports.append(cls) - def recursiveAdd(reportClass: typing.Type[reports.Report]): + def recursiveAdd(reportClass: typing.Type[reports.Report]) -> None: if reportClass.uuid and reportClass.uuid not in alreadyAdded: alreadyAdded.add(reportClass.uuid) addReportCls(reportClass) diff --git a/server/src/uds/reports/lists/__init__.py b/server/src/uds/reports/lists/__init__.py index e774bbb00..2cb963d21 100644 --- a/server/src/uds/reports/lists/__init__.py +++ b/server/src/uds/reports/lists/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, diff --git a/server/src/uds/reports/lists/base.py b/server/src/uds/reports/lists/base.py index dc228deb4..822d6b837 100644 --- a/server/src/uds/reports/lists/base.py +++ b/server/src/uds/reports/lists/base.py @@ -38,6 +38,6 @@ from uds.core import reports class ListReport(reports.Report): group = _('Lists') # So we can make submenus with reports - def generate(self) -> typing.Union[str, bytes]: + def generate(self) -> bytes: raise NotImplementedError('ListReport generate invoked and not implemented') diff --git a/server/src/uds/reports/lists/users.py b/server/src/uds/reports/lists/users.py index 57612c3be..519018ac9 100644 --- a/server/src/uds/reports/lists/users.py +++ b/server/src/uds/reports/lists/users.py @@ -64,7 +64,7 @@ class ListReportUsers(ListReport): description = _('List users of platform') # Report description uuid = '8cd1cfa6-ed48-11e4-83e5-10feed05884b' - def initGui(self): + def initGui(self) -> None: logger.debug('Initializing gui') vals = [ gui.choiceItem(v.uuid, v.name) for v in Authenticator.objects.all() @@ -72,7 +72,7 @@ class ListReportUsers(ListReport): self.authenticator.setValues(vals) - def generate(self): + def generate(self) -> bytes: auth = Authenticator.objects.get(uuid=self.authenticator.value) users = auth.users.order_by('name') @@ -101,7 +101,7 @@ class ListReportsUsersCSV(ListReportUsers): auth = Authenticator.objects.get(uuid=self.authenticator.value) self.filename = auth.name + '.csv' - def generate(self): + def generate(self) -> bytes: output = io.StringIO() writer = csv.writer(output) auth = Authenticator.objects.get(uuid=self.authenticator.value) @@ -114,4 +114,4 @@ class ListReportsUsersCSV(ListReportUsers): # writer.writerow(['ñoño', 'ádios', 'hola']) - return output.getvalue() + return output.getvalue().encode() diff --git a/server/src/uds/reports/stats/__init__.py b/server/src/uds/reports/stats/__init__.py index 2da76ae38..660c76efd 100644 --- a/server/src/uds/reports/stats/__init__.py +++ b/server/src/uds/reports/stats/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, diff --git a/server/src/uds/reports/stats/auth_stats.py b/server/src/uds/reports/stats/auth_stats.py index c2fdb18d5..ce857069d 100644 --- a/server/src/uds/reports/stats/auth_stats.py +++ b/server/src/uds/reports/stats/auth_stats.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, diff --git a/server/src/uds/reports/stats/base.py b/server/src/uds/reports/stats/base.py index 44f33e164..52c9e0a6d 100644 --- a/server/src/uds/reports/stats/base.py +++ b/server/src/uds/reports/stats/base.py @@ -39,7 +39,7 @@ from ..auto import ReportAuto class StatsReport(reports.Report): group = _('Statistics') # So we can make submenus with reports - def generate(self) -> typing.Union[str, bytes]: + def generate(self) -> bytes: raise NotImplementedError('StatsReport generate invoked and not implemented') diff --git a/server/src/uds/reports/stats/pool_users_summary.py b/server/src/uds/reports/stats/pool_users_summary.py index 7aba1801a..8362d4649 100644 --- a/server/src/uds/reports/stats/pool_users_summary.py +++ b/server/src/uds/reports/stats/pool_users_summary.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -77,10 +76,7 @@ class UsageSummaryByUsersPool(StatsReport): required=True ) - def initialize(self, values): - pass - - def initGui(self): + def initGui(self) -> None: logger.debug('Initializing gui') vals = [ gui.choiceItem(v.uuid, v.name) for v in ServicePool.objects.all() @@ -130,7 +126,7 @@ class UsageSummaryByUsersPool(StatsReport): def getData(self) -> typing.Tuple[typing.List[typing.Dict[str, typing.Any]], str]: return self.getPoolData(ServicePool.objects.get(uuid=self.pool.value)) - def generate(self): + def generate(self) -> bytes: items, poolName = self.getData() return self.templateAsPDF( @@ -157,7 +153,7 @@ class UsageSummaryByUsersPoolCSV(UsageSummaryByUsersPool): startDate = UsageSummaryByUsersPool.startDate endDate = UsageSummaryByUsersPool.endDate - def generate(self): + def generate(self) -> bytes: output = io.StringIO() writer = csv.writer(output) @@ -168,4 +164,4 @@ class UsageSummaryByUsersPoolCSV(UsageSummaryByUsersPool): for v in reportData: writer.writerow([v['user'], v['sessions'], v['hours'], v['average']]) - return output.getvalue() + return output.getvalue().encode() diff --git a/server/src/uds/reports/stats/pools_performance.py b/server/src/uds/reports/stats/pools_performance.py index 8a3ca0290..e702308e1 100644 --- a/server/src/uds/reports/stats/pools_performance.py +++ b/server/src/uds/reports/stats/pools_performance.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -63,10 +63,7 @@ class PoolPerformanceReport(StatsReport): # Input fields pools = gui.MultiChoiceField( - order=1, - label=_('Pools'), - tooltip=_('Pools for report'), - required=True + order=1, label=_('Pools'), tooltip=_('Pools for report'), required=True ) startDate = gui.DateField( @@ -74,7 +71,7 @@ class PoolPerformanceReport(StatsReport): label=_('Starting date'), tooltip=_('starting date for report'), defvalue=datetime.date.min, - required=True + required=True, ) endDate = gui.DateField( @@ -82,7 +79,7 @@ class PoolPerformanceReport(StatsReport): label=_('Finish date'), tooltip=_('finish date for report'), defvalue=datetime.date.max, - required=True + required=True, ) samplingPoints = gui.NumericField( @@ -92,28 +89,31 @@ class PoolPerformanceReport(StatsReport): minValue=0, maxValue=32, tooltip=_('Number of sampling points used in charts'), - defvalue='8' + defvalue='8', ) - def initialize(self, values): - pass - - def initGui(self): + def initGui(self) -> None: logger.debug('Initializing gui') vals = [ - gui.choiceItem(v.uuid, v.name) for v in ServicePool.objects.all().order_by('name') + gui.choiceItem(v.uuid, v.name) + for v in ServicePool.objects.all().order_by('name') ] self.pools.setValues(vals) - def getPools(self) -> typing.List[typing.Tuple[str, str]]: - return [(v.id, v.name) for v in ServicePool.objects.filter(uuid__in=self.pools.value)] + def getPools(self) -> typing.Iterable[typing.Tuple[str, str]]: + for p in ServicePool.objects.filter(uuid__in=self.pools.value): + yield (str(p.id), p.name) - def getRangeData(self) -> typing.Tuple[str, typing.List, typing.List]: # pylint: disable=too-many-locals + def getRangeData( + self, + ) -> typing.Tuple[str, typing.List, typing.List]: # pylint: disable=too-many-locals start = self.startDate.stamp() end = self.endDate.stamp() if self.samplingPoints.num() < 2: - self.samplingPoints.value = (self.endDate.date() - self.startDate.date()).days + self.samplingPoints.value = ( + self.endDate.date() - self.startDate.date() + ).days if self.samplingPoints.num() < 2: self.samplingPoints.value = 2 if self.samplingPoints.num() > 32: @@ -155,7 +155,18 @@ class PoolPerformanceReport(StatsReport): dataAccesses = [] for interval in samplingIntervals: key = (interval[0] + interval[1]) / 2 - q = events.statsManager().getEvents(events.OT_DEPLOYED, events.ET_ACCESS, since=interval[0], to=interval[1], owner_id=p[0]).values(fld).annotate(cnt=Count(fld)) + q = ( + events.statsManager() + .getEvents( + events.OT_DEPLOYED, + events.ET_ACCESS, + since=interval[0], + to=interval[1], + owner_id=p[0], + ) + .values(fld) + .annotate(cnt=Count(fld)) + ) accesses = 0 for v in q: accesses += v['cnt'] @@ -165,17 +176,21 @@ class PoolPerformanceReport(StatsReport): reportData.append( { 'name': p[1], - 'date': tools.timestampAsStr(interval[0], xLabelFormat) + ' - ' + tools.timestampAsStr(interval[1], xLabelFormat), + 'date': tools.timestampAsStr(interval[0], xLabelFormat) + + ' - ' + + tools.timestampAsStr(interval[1], xLabelFormat), 'users': len(q), - 'accesses': accesses + 'accesses': accesses, } ) - poolsData.append({ - 'pool': p[0], - 'name': p[1], - 'dataUsers': dataUsers, - 'dataAccesses': dataAccesses, - }) + poolsData.append( + { + 'pool': p[0], + 'name': p[1], + 'dataUsers': dataUsers, + 'dataAccesses': dataAccesses, + } + ) return xLabelFormat, poolsData, reportData @@ -194,13 +209,17 @@ class PoolPerformanceReport(StatsReport): data = { 'title': _('Distinct Users'), 'x': X, - 'xtickFnc': lambda l: filters.date(datetime.datetime.fromtimestamp(X[int(l)]), xLabelFormat) if int(l) >= 0 else '', + 'xtickFnc': lambda l: filters.date( + datetime.datetime.fromtimestamp(X[int(l)]), xLabelFormat + ) + if int(l) >= 0 + else '', 'xlabel': _('Date'), - 'y': [{ - 'label': p['name'], - 'data': [v[1] for v in p['dataUsers']] - } for p in poolsData], - 'ylabel': _('Users') + 'y': [ + {'label': p['name'], 'data': [v[1] for v in p['dataUsers']]} + for p in poolsData + ], + 'ylabel': _('Users'), } graphs.barChart(SIZE, data, graph1) @@ -209,13 +228,17 @@ class PoolPerformanceReport(StatsReport): data = { 'title': _('Accesses'), 'x': X, - 'xtickFnc': lambda l: filters.date(datetime.datetime.fromtimestamp(X[int(l)]), xLabelFormat) if int(l) >= 0 else '', + 'xtickFnc': lambda l: filters.date( + datetime.datetime.fromtimestamp(X[int(l)]), xLabelFormat + ) + if int(l) >= 0 + else '', 'xlabel': _('Date'), - 'y': [{ - 'label': p['name'], - 'data': [v[1] for v in p['dataAccesses']] - } for p in poolsData], - 'ylabel': _('Accesses') + 'y': [ + {'label': p['name'], 'data': [v[1] for v in p['dataAccesses']]} + for p in poolsData + ], + 'ylabel': _('Accesses'), } graphs.barChart(SIZE, data, graph2) @@ -255,7 +278,14 @@ class PoolPerformanceReportCSV(PoolPerformanceReport): reportData = self.getRangeData()[2] - writer.writerow([ugettext('Pool'), ugettext('Date range'), ugettext('Users'), ugettext('Accesses')]) + writer.writerow( + [ + ugettext('Pool'), + ugettext('Date range'), + ugettext('Users'), + ugettext('Accesses'), + ] + ) for v in reportData: writer.writerow([v['name'], v['date'], v['users'], v['accesses']]) diff --git a/server/src/uds/reports/stats/pools_usage_day.py b/server/src/uds/reports/stats/pools_usage_day.py index f9974a18d..2a975b3b2 100644 --- a/server/src/uds/reports/stats/pools_usage_day.py +++ b/server/src/uds/reports/stats/pools_usage_day.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -65,14 +64,11 @@ class CountersPoolAssigned(StatsReport): label=_('Date'), tooltip=_('Date for report'), defvalue='', - required=True + required=True, ) pools = gui.MultiChoiceField( - order=1, - label=_('Pools'), - tooltip=_('Pools for report'), - required=True + order=1, label=_('Pools'), tooltip=_('Pools for report'), required=True ) def initialize(self, values): @@ -81,7 +77,8 @@ class CountersPoolAssigned(StatsReport): def initGui(self): logger.debug('Initializing gui') vals = [ - gui.choiceItem(v.uuid, v.name) for v in ServicePool.objects.all().order_by('name') + gui.choiceItem(v.uuid, v.name) + for v in ServicePool.objects.all().order_by('name') ] self.pools.setValues(vals) @@ -101,13 +98,21 @@ class CountersPoolAssigned(StatsReport): hours = [0] * 24 - for x in counters.getCounters(pool, counters.CT_ASSIGNED, since=start, to=end, max_intervals=24, use_max=True, all=False): + for x in counters.getCounters( + pool, + counters.CT_ASSIGNED, + since=start, + to=end, + max_intervals=24, + use_max=True, + all=False, + ): hour = x[0].hour val = int(x[1]) if hours[hour] < val: hours[hour] = val - data.append({'uuid':pool.uuid, 'name': pool.name, 'hours': hours}) + data.append({'uuid': pool.uuid, 'name': pool.name, 'hours': hours}) logger.debug('data: %s', data) @@ -122,15 +127,14 @@ class CountersPoolAssigned(StatsReport): d = { 'title': _('Services by hour'), 'x': X, - 'xtickFnc': lambda xx: '{:02d}'.format(xx), # pylint: disable=unnecessary-lambda + 'xtickFnc': lambda xx: '{:02d}'.format( + xx + ), # pylint: disable=unnecessary-lambda 'xlabel': _('Hour'), 'y': [ - { - 'label': i['name'], - 'data': [i['hours'][v] for v in X] - } for i in items + {'label': i['name'], 'data': [i['hours'][v] for v in X]} for i in items ], - 'ylabel': 'Services' + 'ylabel': 'Services', } graphs.barChart(SIZE, d, graph1) @@ -139,7 +143,10 @@ class CountersPoolAssigned(StatsReport): 'uds/reports/stats/pools-usage-day.html', dct={ 'data': items, - 'pools': [v.name for v in ServicePool.objects.filter(uuid__in=self.pools.value)], + 'pools': [ + v.name + for v in ServicePool.objects.filter(uuid__in=self.pools.value) + ], 'beginning': self.startDate.date(), }, header=ugettext('Services usage report for a day'), diff --git a/server/src/uds/reports/stats/pools_usage_summary.py b/server/src/uds/reports/stats/pools_usage_summary.py index 4f5cee0d1..717041faa 100644 --- a/server/src/uds/reports/stats/pools_usage_summary.py +++ b/server/src/uds/reports/stats/pools_usage_summary.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2020 Virtual Cable S.L. +# Copyright (c) 2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -42,10 +41,13 @@ from .usage_by_pool import UsageByPool logger = logging.getLogger(__name__) + class PoolsUsageSummary(UsageByPool): filename = 'summary_pools_usage.pdf' name = _('Summary of pools usage') # Report name - description = _('Summary of Pools usage with time totals, accesses totals, time total by pool') # Report description + description = _( + 'Summary of Pools usage with time totals, accesses totals, time total by pool' + ) # Report description uuid = 'aba55fe5-c4df-5240-bbe6-36340220cb5d' # Input fields @@ -53,7 +55,11 @@ class PoolsUsageSummary(UsageByPool): startDate = UsageByPool.startDate endDate = UsageByPool.endDate - def getData(self): + def processedData( + self, + ) -> typing.Tuple[ + typing.ValuesView[typing.MutableMapping[str, typing.Any]], int, int, int + ]: orig, poolNames = super().getData() pools: typing.Dict[str, typing.Dict] = {} @@ -69,7 +75,7 @@ class PoolsUsageSummary(UsageByPool): 'name': v['pool_name'], 'time': 0, 'count': 0, - 'users': set() + 'users': set(), } pools[uuid]['time'] += v['time'] pools[uuid]['count'] += 1 @@ -88,7 +94,7 @@ class PoolsUsageSummary(UsageByPool): return pools.values(), totalTime, totalCount or 1, len(uniqueUsers) def generate(self): - pools, totalTime, totalCount, uniqueUsers = self.getData() + pools, totalTime, totalCount, uniqueUsers = self.processedData() start = self.startDate.value end = self.endDate.value @@ -104,7 +110,9 @@ class PoolsUsageSummary(UsageByPool): 'time': str(datetime.timedelta(seconds=p['time'])), 'count': p['count'], 'users': p['users'], - 'mean': str(datetime.timedelta(seconds=p['time'] // int(p['count']))), + 'mean': str( + datetime.timedelta(seconds=p['time'] // int(p['count'])) + ), } for p in pools ), @@ -112,14 +120,20 @@ class PoolsUsageSummary(UsageByPool): 'count': totalCount, 'users': uniqueUsers, 'mean': str(datetime.timedelta(seconds=totalTime // totalCount)), - 'start': start, 'end': end, }, - header=ugettext('Summary of Pools usage') + ' ' + start + ' ' + ugettext('to') + ' ' + end, - water=ugettext('UDS Report Summary of pools usage') + header=ugettext('Summary of Pools usage') + + ' ' + + start + + ' ' + + ugettext('to') + + ' ' + + end, + water=ugettext('UDS Report Summary of pools usage'), ) + class PoolsUsageSummaryCSV(PoolsUsageSummary): filename = 'summary_pools_usage.csv' mime_type = 'text/csv' # Report returns pdfs by default, but could be anything else @@ -135,13 +149,31 @@ class PoolsUsageSummaryCSV(PoolsUsageSummary): output = io.StringIO() writer = csv.writer(output) - reportData, totalTime, totalCount, totalUsers = self.getData() + reportData, totalTime, totalCount, totalUsers = self.processedData() - writer.writerow([ugettext('Pool'), ugettext('Total Time (seconds)'), ugettext('Total Accesses'), ugettext('Unique users'), ugettext('Mean time (seconds)')]) + writer.writerow( + [ + ugettext('Pool'), + ugettext('Total Time (seconds)'), + ugettext('Total Accesses'), + ugettext('Unique users'), + ugettext('Mean time (seconds)'), + ] + ) for v in reportData: - writer.writerow([v['name'], v['time'], v['count'], v['users'], v['time'] // v['count']]) + writer.writerow( + [v['name'], v['time'], v['count'], v['users'], v['time'] // v['count']] + ) - writer.writerow([ugettext('Total'), totalTime, totalCount, totalUsers, totalTime // totalCount]) + writer.writerow( + [ + ugettext('Total'), + totalTime, + totalCount, + totalUsers, + totalTime // totalCount, + ] + ) return output.getvalue() diff --git a/server/src/uds/reports/stats/usage.py b/server/src/uds/reports/stats/usage.py index 083a4bcb7..7b4e6504f 100644 --- a/server/src/uds/reports/stats/usage.py +++ b/server/src/uds/reports/stats/usage.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, diff --git a/server/src/uds/reports/stats/usage_by_pool.py b/server/src/uds/reports/stats/usage_by_pool.py index c04428c9b..3f4cb1421 100644 --- a/server/src/uds/reports/stats/usage_by_pool.py +++ b/server/src/uds/reports/stats/usage_by_pool.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -56,10 +55,7 @@ class UsageByPool(StatsReport): # Input fields pool = gui.MultiChoiceField( - order=1, - label=_('Pool'), - tooltip=_('Pool for report'), - required=True + order=1, label=_('Pool'), tooltip=_('Pool for report'), required=True ) startDate = gui.DateField( @@ -67,7 +63,7 @@ class UsageByPool(StatsReport): label=_('Starting date'), tooltip=_('starting date for report'), defvalue=datetime.date.min, - required=True + required=True, ) endDate = gui.DateField( @@ -75,7 +71,7 @@ class UsageByPool(StatsReport): label=_('Finish date'), tooltip=_('finish date for report'), defvalue=datetime.date.max, - required=True + required=True, ) def initialize(self, values): @@ -84,7 +80,8 @@ class UsageByPool(StatsReport): def initGui(self): logger.debug('Initializing gui') vals = [gui.choiceItem('0-0-0-0', ugettext('ALL POOLS'))] + [ - gui.choiceItem(v.uuid, v.name) for v in ServicePool.objects.all().order_by('name') + gui.choiceItem(v.uuid, v.name) + for v in ServicePool.objects.all().order_by('name') ] self.pool.setValues(vals) @@ -99,7 +96,17 @@ class UsageByPool(StatsReport): pools = ServicePool.objects.filter(uuid__in=self.pool.value) data = [] for pool in pools: - items = events.statsManager().getEvents(events.OT_DEPLOYED, (events.ET_LOGIN, events.ET_LOGOUT), owner_id=pool.id, since=start, to=end).order_by('stamp') + items = ( + events.statsManager() + .getEvents( + events.OT_DEPLOYED, + (events.ET_LOGIN, events.ET_LOGOUT), + owner_id=pool.id, + since=start, + to=end, + ) + .order_by('stamp') + ) logins = {} for i in items: @@ -113,14 +120,16 @@ class UsageByPool(StatsReport): stamp = logins[i.fld4] del logins[i.fld4] total = i.stamp - stamp - data.append({ - 'name': i.fld4, - 'origin': i.fld2.split(':')[0], - 'date': datetime.datetime.fromtimestamp(stamp), - 'time': total, - 'pool': pool.uuid, - 'pool_name': pool.name - }) + data.append( + { + 'name': i.fld4, + 'origin': i.fld2.split(':')[0], + 'date': datetime.datetime.fromtimestamp(stamp), + 'time': total, + 'pool': pool.uuid, + 'pool_name': pool.name, + } + ) return data, ','.join([p.name for p in pools]) @@ -134,7 +143,7 @@ class UsageByPool(StatsReport): 'pool': poolName, }, header=ugettext('Users usage list'), - water=ugettext('UDS Report of users usage') + water=ugettext('UDS Report of users usage'), ) @@ -155,9 +164,19 @@ class UsageByPoolCSV(UsageByPool): reportData = self.getData()[0] - writer.writerow([ugettext('Date'), ugettext('User'), ugettext('Seconds'), ugettext('Pool'), ugettext('Origin')]) + writer.writerow( + [ + ugettext('Date'), + ugettext('User'), + ugettext('Seconds'), + ugettext('Pool'), + ugettext('Origin'), + ] + ) for v in reportData: - writer.writerow([v['date'], v['name'], v['time'], v['pool_name'], v['origin']]) + writer.writerow( + [v['date'], v['name'], v['time'], v['pool_name'], v['origin']] + ) return output.getvalue() diff --git a/server/src/uds/reports/stats/user_access.py b/server/src/uds/reports/stats/user_access.py index 87a1549c7..6017b7f8e 100644 --- a/server/src/uds/reports/stats/user_access.py +++ b/server/src/uds/reports/stats/user_access.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - # -# Copyright (c) 2015-2019 Virtual Cable S.L. +# Copyright (c) 2015-2020 Virtual Cable S.L.U. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -66,7 +65,7 @@ class StatsReportLogin(StatsReport): label=_('Starting date'), tooltip=_('starting date for report'), defvalue=datetime.date.min, - required=True + required=True, ) endDate = gui.DateField( @@ -74,7 +73,7 @@ class StatsReportLogin(StatsReport): label=_('Finish date'), tooltip=_('finish date for report'), defvalue=datetime.date.max, - required=True + required=True, ) samplingPoints = gui.NumericField( @@ -84,7 +83,7 @@ class StatsReportLogin(StatsReport): minValue=0, maxValue=128, tooltip=_('Number of sampling points used in charts'), - defvalue='64' + defvalue='64', ) def initialize(self, values): @@ -97,7 +96,9 @@ class StatsReportLogin(StatsReport): start = self.startDate.stamp() end = self.endDate.stamp() if self.samplingPoints.num() < 8: - self.samplingPoints.value = (self.endDate.date() - self.startDate.date()).days + self.samplingPoints.value = ( + self.endDate.date() - self.startDate.date() + ).days if self.samplingPoints.num() < 2: self.samplingPoints.value = 2 if self.samplingPoints.num() > 128: @@ -124,12 +125,23 @@ class StatsReportLogin(StatsReport): reportData = [] for interval in samplingIntervals: key = (interval[0] + interval[1]) / 2 - val = events.statsManager().getEvents(events.OT_AUTHENTICATOR, events.ET_LOGIN, since=interval[0], to=interval[1]).count() + val = ( + events.statsManager() + .getEvents( + events.OT_AUTHENTICATOR, + events.ET_LOGIN, + since=interval[0], + to=interval[1], + ) + .count() + ) data.append((key, val)) # @UndefinedVariable reportData.append( { - 'date': tools.timestampAsStr(interval[0], xLabelFormat) + ' - ' + tools.timestampAsStr(interval[1], xLabelFormat), - 'users': val + 'date': tools.timestampAsStr(interval[0], xLabelFormat) + + ' - ' + + tools.timestampAsStr(interval[1], xLabelFormat), + 'users': val, } ) @@ -142,7 +154,9 @@ class StatsReportLogin(StatsReport): dataWeek = [0] * 7 dataHour = [0] * 24 dataWeekHour = [[0] * 24 for _ in range(7)] - for val in events.statsManager().getEvents(events.OT_AUTHENTICATOR, events.ET_LOGIN, since=start, to=end): + for val in events.statsManager().getEvents( + events.OT_AUTHENTICATOR, events.ET_LOGIN, since=start, to=end + ): s = datetime.datetime.fromtimestamp(val.stamp) dataWeek[s.weekday()] += 1 dataHour[s.hour] += 1 @@ -170,16 +184,13 @@ class StatsReportLogin(StatsReport): d = { 'title': _('Users Access (global)'), 'x': X, - 'xtickFnc': lambda l: filters.date(datetime.datetime.fromtimestamp(l), xLabelFormat), + 'xtickFnc': lambda l: filters.date( + datetime.datetime.fromtimestamp(l), xLabelFormat + ), 'xlabel': _('Date'), - 'y': [ - { - 'label': 'Users', - 'data': [v[1] for v in data] - } - ], + 'y': [{'label': 'Users', 'data': [v[1] for v in data]}], 'ylabel': 'Users', - 'allTicks': False + 'allTicks': False, } graphs.lineChart(SIZE, d, graph1) @@ -193,15 +204,18 @@ class StatsReportLogin(StatsReport): d = { 'title': _('Users Access (by week)'), 'x': X, - 'xtickFnc': lambda l: [_('Monday'), _('Tuesday'), _('Wednesday'), _('Thursday'), _('Friday'), _('Saturday'), _('Sunday')][l], + 'xtickFnc': lambda l: [ + _('Monday'), + _('Tuesday'), + _('Wednesday'), + _('Thursday'), + _('Friday'), + _('Saturday'), + _('Sunday'), + ][l], 'xlabel': _('Day of week'), - 'y': [ - { - 'label': 'Users', - 'data': [v for v in dataWeek] - } - ], - 'ylabel': 'Users' + 'y': [{'label': 'Users', 'data': [v for v in dataWeek]}], + 'ylabel': 'Users', } graphs.barChart(SIZE, d, graph2) @@ -211,13 +225,8 @@ class StatsReportLogin(StatsReport): 'title': _('Users Access (by hour)'), 'x': X, 'xlabel': _('Hour'), - 'y': [ - { - 'label': 'Users', - 'data': [v for v in dataHour] - } - ], - 'ylabel': 'Users' + 'y': [{'label': 'Users', 'data': [v for v in dataHour]}], + 'ylabel': 'Users', } graphs.barChart(SIZE, d, graph3) @@ -231,10 +240,17 @@ class StatsReportLogin(StatsReport): 'xtickFnc': lambda l: l, 'y': Y, 'ylabel': _('Day of week'), - 'ytickFnc': lambda l: [_('Monday'), _('Tuesday'), _('Wednesday'), _('Thursday'), _('Friday'), _('Saturday'), _('Sunday')][l], + 'ytickFnc': lambda l: [ + _('Monday'), + _('Tuesday'), + _('Wednesday'), + _('Thursday'), + _('Friday'), + _('Saturday'), + _('Sunday'), + ][l], 'z': dataWeekHour, - 'zlabel': _('Users') - + 'zlabel': _('Users'), } graphs.surfaceChart(SIZE, d, graph4) @@ -249,7 +265,12 @@ class StatsReportLogin(StatsReport): }, header=ugettext('Users access to UDS'), water=ugettext('UDS Report for users access'), - images={'graph1': graph1.getvalue(), 'graph2': graph2.getvalue(), 'graph3': graph3.getvalue(), 'graph4': graph4.getvalue()}, + images={ + 'graph1': graph1.getvalue(), + 'graph2': graph2.getvalue(), + 'graph3': graph3.getvalue(), + 'graph4': graph4.getvalue(), + }, )