burmatovan/openuds
1
0
Fork 0
forked from shaba/openuds
Code Issues Pull Requests Projects Releases Wiki Activity

Improved user services queries a LOT :)

Browse Source
This commit is contained in:
Adolfo Gómez García 2020-05-22 03:18:04 +02:00
parent 42b06ff688
commit aee16dd075
5 changed files with 120 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/src/uds/core/util/calendar/__init__.py
View File

@ -131,7 +131,6 @@ class CalendarChecker:
"""
Checks if the given time is a valid event on calendar
@param dtime: Datetime object to check
TODO: We can improve performance of this by getting from a cache first if we can
"""
if dtime is None:
dtime = getSqlDatetime()

29
server/src/uds/models/meta_pool.py
View File

@ -53,6 +53,7 @@ from .calendar import Calendar
if typing.TYPE_CHECKING:
import datetime
from .user import User
logger = logging.getLogger(__name__)
@ -106,6 +107,7 @@ class MetaPool(UUIDModel, TaggingMixin): # type: ignore
bool -- [description]
"""
total, maintenance = 0, 0
p: ServicePool
for p in self.pools.all():
total += 1
if p.isInMaintenance():
@ -121,7 +123,7 @@ class MetaPool(UUIDModel, TaggingMixin): # type: ignore
access = self.fallbackAccess
# Let's see if we can access by current datetime
for ac in self.calendarAccess.order_by('priority'):
for ac in sorted(self.calendarAccess.all(), key=lambda x: x.priority):
if CalendarChecker(ac.calendar).check(chkDateTime) is True:
access = ac.access
break # Stops on first rule match found
@ -136,7 +138,7 @@ class MetaPool(UUIDModel, TaggingMixin): # type: ignore
return self.name
@staticmethod
def getForGroups(groups) -> QuerySet:
def getForGroups(groups: typing.Iterable['Group'], user: typing.Optional['User'] = None) -> QuerySet:
"""
Return deployed services with publications for the groups requested.
@ -151,7 +153,30 @@ class MetaPool(UUIDModel, TaggingMixin): # type: ignore
assignedGroups__in=groups,
assignedGroups__state=states.group.ACTIVE,
visible=True
).prefetch_related(
'servicesPoolGroup',
'servicesPoolGroup__image',
'assignedGroups',
'assignedGroups',
'accessCalendars',
'accessCalendars__rules',
'pools',
'pools__service',
'pools__service__provider',
'pools__image',
'pools__transports',
'pools__transports__networks'
)
if user:
meta = meta.annotate(
number_assignations=models.Count(
'pools__userServices',
filter=models.Q(
pools__userServices__user=user,
pools__userServices__in_use=True
)
)
)
# TODO: Maybe we can exclude non "usable" metapools (all his pools are in maintenance mode?)
return meta

67
server/src/uds/models/service_pool.py
View File

@ -216,7 +216,7 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
return False
def isInMaintenance(self) -> bool:
return self.service is not None and self.service.isInMaintenance()
return self.service.isInMaintenance() if self.service else True
def isVisible(self) -> bool:
return self.visible
@ -255,7 +255,7 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
access = self.fallbackAccess
# Let's see if we can access by current datetime
for ac in self.calendarAccess.order_by('priority'):
for ac in sorted(self.calendarAccess.all(), key=lambda x:x.priority):
if CalendarChecker(ac.calendar).check(chkDateTime) is True:
access = ac.access
break # Stops on first rule match found
@ -429,8 +429,8 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
self.validatePublication()
@staticmethod
def getDeployedServicesForGroups(groups) -> typing.List['ServicePool']:
"""
def getDeployedServicesForGroups(groups: typing.Iterable['Group'], user: typing.Optional['User'] = None) -> typing.List['ServicePool']:
"""
Return deployed services with publications for the groups requested.
Args:
@ -441,15 +441,68 @@ class ServicePool(UUIDModel, TaggingMixin): # type: ignore
"""
from uds.core import services
# Get services that HAS publications
list1 = ServicePool.objects.filter(
list1 = (
ServicePool.objects.filter(
assignedGroups__in=groups,
assignedGroups__state=states.group.ACTIVE,
state=states.servicePool.ACTIVE,
visible=True
).distinct().annotate(cuenta=models.Count('publications')).exclude(cuenta=0)
).distinct().annotate(cuenta=models.Count('publications')).exclude(cuenta=0)
.prefetch_related(
'transports',
'transports__networks',
'memberOfMeta',
'osmanager',
'publications',
'servicesPoolGroup',
'servicesPoolGroup__image',
'service',
'service__provider',
'calendarAccess',
'calendarAccess__calendar',
'calendarAccess__calendar__rules',
'image'
)
)
# Now get deployed services that DO NOT NEED publication
doNotNeedPublishing = [t.type() for t in services.factory().servicesThatDoNotNeedPublication()]
list2 = ServicePool.objects.filter(assignedGroups__in=groups, assignedGroups__state=states.group.ACTIVE, service__data_type__in=doNotNeedPublishing, state=states.servicePool.ACTIVE, visible=True)
list2 = (
ServicePool.objects.filter(
assignedGroups__in=groups,
assignedGroups__state=states.group.ACTIVE,
service__data_type__in=doNotNeedPublishing,
state=states.servicePool.ACTIVE,
visible=True
)
.prefetch_related(
'transports',
'transports__networks',
'memberOfMeta',
'servicesPoolGroup',
'servicesPoolGroup__image',
'service',
'service__provider',
'calendarAccess',
'calendarAccess__calendar',
'calendarAccess__calendar__rules',
'image'
)
)
if user: # Optimize loading if there is some assgned service..
list1 = list1.annotate(
number_assignations=models.Count(
'userServices', filter=models.Q(
userServices__user=user, userServices__in_use=True
)
)
)
list2 = list2.annotate(
number_assignations=models.Count(
'userServices', filter=models.Q(
userServices__user=user, userServices__in_use=True
)
)
)
# And generate a single list without duplicates
return list(set([r for r in list1] + [r for r in list2]))

19
server/src/uds/models/user.py
View File

@ -34,7 +34,7 @@ import logging
import typing
from django.db import models
from django.db.models import signals
from django.db.models import signals, Q, Count
from uds.core.util import log
@ -145,9 +145,9 @@ class User(UUIDModel):
"""
returns the groups (and metagroups) this user belongs to
"""
if self.parent is not None and self.parent != '':
if self.parent:
try:
usr = User.objects.get(uuid=self.parent)
usr = User.objects.prefetch_related('groups').get(uuid=self.parent)
except Exception: # If parent do not exists
usr = self
else:
@ -160,18 +160,21 @@ class User(UUIDModel):
yield g
# Locate metagroups
for g in self.manager.groups.filter(is_meta=True):
numberGroupsBelongingInMeta: int = g.groups.filter(id__in=grps).count()
for g in (self.manager.groups.filter(is_meta=True)
.annotate(number_groups=Count('groups')) # g.groups.count()
.annotate(number_belongs_meta=Count('groups', filter=Q(groups__id__in=grps))) # g.groups.filter(id__in=grps).count()
):
numberGroupsBelongingInMeta: int = g.number_belongs_meta
logger.debug('gn = %s', numberGroupsBelongingInMeta)
logger.debug('groups count: %s', g.groups.count())
logger.debug('groups count: %s', g.number_groups)
if g.meta_if_any is True and numberGroupsBelongingInMeta > 0:
numberGroupsBelongingInMeta = g.groups.count()
numberGroupsBelongingInMeta = g.number_groups
logger.debug('gn after = %s', numberGroupsBelongingInMeta)
if numberGroupsBelongingInMeta == g.groups.count(): # If a meta group is empty, all users belongs to it. we can use gn != 0 to check that if it is empty, is not valid
if numberGroupsBelongingInMeta == g.number_groups: # If a meta group is empty, all users belongs to it. we can use gn != 0 to check that if it is empty, is not valid
# This group matches
yield g

40
server/src/uds/web/util/services.py
View File

@ -35,7 +35,7 @@ from django.utils.translation import ugettext
from django.utils import formats
from django.urls.base import reverse
from uds.models import ServicePool, Transport, Network, ServicePoolGroup, MetaPool
from uds.models import ServicePool, Transport, Network, ServicePoolGroup, MetaPool, getSqlDatetime
from uds.core.util.config import GlobalConfig
from uds.core.util import html
@ -69,8 +69,9 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
# We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds
groups = list(request.user.getGroups())
availServicePools = ServicePool.getDeployedServicesForGroups(groups)
availMetaPools = MetaPool.getForGroups(groups)
availServicePools = list(ServicePool.getDeployedServicesForGroups(groups, request.user)) # Pass in user to get "number_assigned" to optimize
availMetaPools = list(MetaPool.getForGroups(groups, request.user)) # Pass in user to get "number_assigned" to optimize
now = getSqlDatetime()
# Information for administrators
nets = ''
@ -81,7 +82,8 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
if request.user.isStaff():
nets = ','.join([n.name for n in Network.networksFor(request.ip)])
tt = []
for t in Transport.objects.all():
t: Transport
for t in Transport.objects.all().prefetch_related('networks'):
if t.validForIp(request.ip):
tt.append(t.name)
validTrans = ','.join(tt)
@ -89,9 +91,10 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
logger.debug('Checking meta pools: %s', availMetaPools)
services = []
meta: MetaPool
# Preload all assigned user services for this user
# Add meta pools data first
for meta in availMetaPools:
# Check that we have access to at least one transport on some of its children
hasUsablePools = False
in_use = False
@ -104,7 +107,7 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
hasUsablePools = True
break
if not in_use:
if not in_use and meta.number_assignations: # Only look for assignation on possible used
assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user)
if assignedUserService:
in_use = assignedUserService.in_use
@ -134,7 +137,7 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
'allow_users_remove': False,
'allow_users_reset': False,
'maintenance': meta.isInMaintenance(),
'not_accesible': not meta.isAccessAllowed(),
'not_accesible': not meta.isAccessAllowed(now),
'in_use': in_use,
'to_be_replaced': None,
'to_be_replaced_text': '',
@ -142,13 +145,14 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
})
# Now generic user service
svr: ServicePool
for svr in availServicePools:
# Skip pools that are part of meta pools
if svr.is_meta:
continue
trans = []
for t in svr.transports.all().order_by('priority'):
for t in sorted(svr.transports.all(), key=lambda x: x.priority): # In memory sort, allows reuse prefetched and not too big array
typeTrans = t.getType()
if typeTrans is None: # This may happen if we "remove" a transport type but we have a transport of that kind on DB
continue
@ -170,17 +174,17 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
if not trans:
continue
if svr.image is not None:
if svr.image:
imageId = svr.image.uuid
else:
imageId = 'x'
# Locate if user service has any already assigned user service for this
ads = userServiceManager().getExistingAssignationForUser(svr, request.user)
if ads is None:
in_use = False
else:
in_use = ads.in_use
# Locate if user service has any already assigned user service for this. Use "pre cached" number of assignations in this pool to optimize
in_use = False
if svr.number_assignations: # Anotated value got from getDeployedServicesForGroups(...). If 0, no assignation for this user
ads = userServiceManager().getExistingAssignationForUser(svr, request.user)
if ads:
in_use = ads.in_use
group = svr.servicesPoolGroup.as_dict if svr.servicesPoolGroup else ServicePoolGroup.default().as_dict
@ -203,20 +207,20 @@ def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: #
'allow_users_remove': svr.allow_users_remove,
'allow_users_reset': svr.allow_users_reset,
'maintenance': svr.isInMaintenance(),
'not_accesible': not svr.isAccessAllowed(),
'not_accesible': not svr.isAccessAllowed(now),
'in_use': in_use,
'to_be_replaced': tbr,
'to_be_replaced_text': tbrt,
'custom_calendar_text': svr.calendar_message,
})
logger.debug('Services: %s', services)
# logger.debug('Services: %s', services)
# Sort services and remove services with no transports...
services = [s for s in sorted(services, key=lambda s: s['name'].upper()) if s['transports']]
autorun = False
if len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(True) and services[0]['transports']:
if len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(False) and services[0]['transports']:
if request.session.get('autorunDone', '0') == '0':
request.session['autorunDone'] = '1'
autorun = True