added initial export command for relevant UDS entities

server/src/uds/REST/methods/users_groups.py

@@ -104,7 +104,7 @@ class Users(DetailHandler):
                             'is_admin',
                             'last_access',
                             'parent',
-                            'mfaData',
+                            'mfa_data',
                         ))
                     )
                 )
@@ -128,7 +128,7 @@ class Users(DetailHandler):
                         'is_admin',
                         'last_access',
                         'parent',
-                        'mfaData',
+                        'mfa_data',
                     ),
                 )
                 res['id'] = u.uuid
@@ -207,9 +207,9 @@ class Users(DetailHandler):
             valid_fields.append('password')
             self._params['password'] = cryptoManager().hash(self._params['password'])
         
-        if 'mfaData' in self._params:
+        if 'mfa_data' in self._params:
-            valid_fields.append('mfaData')
+            valid_fields.append('mfa_data')
-            self._params['mfaData'] = self._params['mfaData'].strip()
+            self._params['mfa_data'] = self._params['mfa_data'].strip()
 
         fields = self.readFieldsFromParams(valid_fields)
         if not self._user.is_admin:

server/src/uds/auths/InternalDB/authenticator.py

@@ -107,7 +107,7 @@ class InternalDBAuth(auths.Authenticator):
 
     def mfaIdentifier(self, username: str) -> str:
         try:
-            return self.dbAuthenticator().users.get(name=username, state=State.ACTIVE).mfaData
+            return self.dbAuthenticator().users.get(name=username, state=State.ACTIVE).mfa_data
         finally:
             return ''
 

server/src/uds/auths/Radius/authenticator.py

@@ -136,7 +136,7 @@ class RadiusAuth(auths.Authenticator):
         )
 
     def mfaStorageKey(self, username: str) -> str:
-        return 'mfa_' + self.dbAuthenticator().uuid + username
+        return 'mfa_' + str(self.dbAuthenticator().uuid) + username
 
     def mfaIdentifier(self, username: str) -> str:
         return self.storage.getPickle(self.mfaStorageKey(username)) or ''

server/src/uds/management/commands/export.py

@@ -0,0 +1,531 @@
+# -*- coding: utf-8 -*-
+
+#
+# Copyright (c) 2022 Virtual Cable S.L.U.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L.U. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Author: Adolfo Gómez, dkmaster at dkmon dot com
+"""
+import logging
+import typing
+import csv
+import yaml
+
+from django.core.management.base import BaseCommand
+
+from uds import models
+
+if typing.TYPE_CHECKING:
+    import argparse
+    from django.db.models import Model
+    from uds.models.uuid_model import UUIDModel
+
+logger = logging.getLogger(__name__)
+
+ModelType = typing.TypeVar('ModelType', bound='UUIDModel')
+T = typing.TypeVar('T')
+
+
+def uuid_object_exporter(obj: 'UUIDModel') -> typing.Dict[str, typing.Any]:
+    """
+    Exports a uuid model to a dict
+    """
+    return {
+        'uuid': obj.uuid,
+    }
+
+
+def managed_object_exporter(
+    obj: models.ManagedObjectModel,
+) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a managed object to a dict
+    """
+    # Get uuid model
+    m = uuid_object_exporter(obj)
+    # Extend with managed object fields
+    m.update(
+        {
+            'name': obj.name,
+            'comments': obj.comments,
+            'data': obj.data,
+            'data_type': obj.data_type,
+        }
+    )
+    return m
+
+
+def provider_exporter(provider: models.Provider) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a provider to a dict
+    """
+    p = managed_object_exporter(provider)
+    p['maintenance_mode'] = provider.maintenance_mode
+    return p
+
+
+def service_exporter(service: models.Service) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a service to a dict
+    """
+    s = managed_object_exporter(service)
+    s['provider'] = service.provider.uuid
+    s['token'] = service.token
+    return s
+
+
+def authenticator_exporter(
+    authenticator: models.Authenticator,
+) -> typing.Dict[str, typing.Any]:
+    """
+    Exports an authenticator to a dict
+    """
+    a = managed_object_exporter(authenticator)
+    a['priority'] = authenticator.priority
+    a['provider'] = authenticator.small_name
+    a['visible'] = authenticator.visible
+    return a
+
+
+def user_exporter(user: models.User) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a user to a dict
+    """
+    u = uuid_object_exporter(user)
+    u.update(
+        {
+            'manager': user.manager.uuid,
+            'name': user.name,
+            'comments': user.comments,
+            'real_name': user.real_name,
+            'state': user.state,
+            'password': user.password,
+            'mfa_data': user.mfa_data,
+            'staff_member': user.staff_member,
+            'is_admin': user.is_admin,
+            'last_access': user.last_access,
+            'parent': user.parent,
+            'created': user.created,
+            'groups': [g.uuid for g in user.groups.all()],
+        }
+    )
+    return u
+
+
+def group_export(group: models.Group) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a group to a dict
+    """
+    g = uuid_object_exporter(group)
+    g.update(
+        {
+            'manager': group.manager.uuid,
+            'name': group.name,
+            'comments': group.comments,
+            'state': group.state,
+            'is_meta': group.is_meta,
+            'meta_if_any': group.meta_if_any,
+            'created': group.created,
+        }
+    )
+    return g
+
+
+def transport_exporter(transport: models.Transport) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a transport to a dict
+    """
+    t = managed_object_exporter(transport)
+    t.update(
+        {
+            'priority': transport.priority,
+            'nets_positive': transport.nets_positive,
+            'allowed_oss': transport.allowed_oss,
+            'label': transport.label,
+            'networks': [n.uuid for n in transport.networks.all()],
+        }
+    )
+    return t
+
+
+def network_exporter(network: models.Network) -> typing.Dict[str, typing.Any]:
+    """
+    Exports a network to a dict
+    """
+    n = uuid_object_exporter(network)
+    n.update(
+        {
+            'name': network.name,
+            'net_start': network.net_start,
+            'net_end': network.net_end,
+            'net_string': network.net_string,
+        }
+    )
+    return n
+
+
+def osmanager_exporter(osmanager: models.OSManager) -> typing.Dict[str, typing.Any]:
+    """
+    Exports an osmanager to a dict
+    """
+    o = managed_object_exporter(osmanager)
+    return o
+
+
+class Command(BaseCommand):
+    help = 'Export entities from UDS to be imported in another UDS instance'
+
+    VALID_ENTITIES: typing.Mapping[str, typing.Callable[[], str]]
+    verbose: bool = True
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.VALID_ENTITIES = {
+            'providers': self.export_providers,
+            'services': self.export_services,
+            'authenticators': self.export_authenticators,
+            'users': self.export_users,
+            'groups': self.export_groups,
+            'networks': self.export_networks,
+            'transports': self.export_transports,
+            'osmanagers': self.export_osmanagers,
+        }
+
+    def add_arguments(self, parser: 'argparse.ArgumentParser') -> None:
+        # Accepts a list of valid entities to export
+        parser.add_argument(
+            'entities',
+            nargs='+',
+            choices=self.VALID_ENTITIES.keys(),
+            default=self.VALID_ENTITIES.keys(),
+            help='Entities to export',
+        )
+
+        # Output file name (will be appended .csv or .yaml)
+        parser.add_argument(
+            '--output',
+            action='store',
+            dest='output',
+            default='/tmp/export.yaml',
+            help='Output file name. Defaults to /tmp/export.yaml',
+        )
+
+        # Filter ALL entities by name
+        parser.add_argument(
+            '--filter-name',
+            action='store',
+            dest='filter_name',
+            default=None,
+            help='Filter ALL entities by name',
+        )
+
+        # filter ALL entities by uuid
+        parser.add_argument(
+            '--filter-uuid',
+            action='store',
+            dest='filter_uuid',
+            default=None,
+            help='Filter ALL entities by uuid',
+        )
+
+        # quiet mode
+        parser.add_argument(
+            '--quiet',
+            action='store_false',
+            dest='verbose',
+            default=True,
+            help='Quiet mode',
+        )
+
+    def handle(self, *args, **options) -> None:
+        self.verbose = options['verbose']
+
+        if self.verbose:
+            self.stderr.write(f'Exporting entities: {",".join(options["entities"])}')
+
+        # Compose filter name for kwargs
+        filter_kwargs = {}
+
+        if options['filter_name']:
+            filter_kwargs['name__icontains'] = options['filter_name']
+
+        if options['filter_uuid']:
+            filter_kwargs['uuid__icontains'] = options['filter_uuid']
+        
+
+        # some entities are redundant, so remove them from the list
+        entities = self.remove_reduntant_entities(options['entities'])
+
+        # For each entity, export it as yaml to output file
+        with open(options['output'], 'w') as f:
+            for entity in entities:
+                self.stderr.write(f'Exporting {entity}')
+                f.write(self.VALID_ENTITIES[entity](**filter_kwargs))
+                f.write('')
+
+        if self.verbose:
+            self.stderr.write(f'Exported to {options["output"]}')
+
+    def apply_filter(
+        self, model: typing.Type[ModelType], **kwargs: str
+    ) -> typing.Iterable[ModelType]:
+        """
+        Applies a filter to a model
+        """
+        if self.verbose:
+            # Explit xxx__icontains=yyy to xxx=yyy
+            values = [f'{k.split("__")[0]}={v}' for k, v in kwargs.items()]
+            self.stderr.write(f'Filtering {model.__name__} by {",".join(values)}')
+        yield from model.objects.all().filter(**kwargs)
+
+    def output_count(
+        self, message: str, iterable: typing.Iterable[T]
+    ) -> typing.Iterable[T]:
+        """
+        Outputs the count of an iterable
+        """
+        count = 0
+        for v in iterable:
+            count += 1
+            if self.verbose:
+                self.stderr.write(f'{message} {count}', ending='\r')
+            yield v
+
+        if self.verbose:
+            self.stderr.write('\n')  # New line after count
+
+    def export_providers(self, **kwargs: str) -> str:
+        """
+        Exports all providers to a list of dicts
+        """
+        return '# Providers\n' + yaml.safe_dump(
+            [provider_exporter(p) for p in self.apply_filter(models.Provider, **kwargs)]
+        )
+
+    def export_services(self, **kwargs: str) -> str:
+        # First, locate providers for services with the filter
+        services_list = list(
+            self.output_count(
+                'Filtering services', self.apply_filter(models.Service, **kwargs)
+            )
+        )
+        providers_list = set(
+            [
+                s.provider
+                for s in self.output_count('Filtering providers', services_list)
+            ]
+        )
+        # Now, export those providers
+        providers = [
+            provider_exporter(p)
+            for p in self.output_count('Saving providers', providers_list)
+        ]
+
+        # Then, export services with the filter
+        services = [
+            service_exporter(s)
+            for s in self.output_count('Saving services', services_list)
+        ]
+
+        return (
+            '# Providers\n'
+            + yaml.safe_dump(providers)
+            + '# Services\n'
+            + yaml.safe_dump(services)
+        )
+
+    def export_authenticators(self, **kwargs: str) -> str:
+        """
+        Exports all authenticators to a list of dicts
+        """
+        return '# Authenticators\n' + yaml.safe_dump(
+            [
+                authenticator_exporter(a)
+                for a in self.output_count(
+                    'Saving authenticators',
+                    self.apply_filter(models.Authenticator, **kwargs),
+                )
+            ]
+        )
+
+    def export_users(self, **kwargs: str) -> str:
+        """
+        Exports all users to a list of dicts
+        """
+        # first, locate authenticators for users with the filter
+        users_list = list(
+            self.output_count(
+                'Filtering users', self.apply_filter(models.User, **kwargs)
+            )
+        )
+        authenticators_list = set(
+            [
+                u.manager
+                for u in self.output_count('Filtering authenticators', users_list)
+            ]
+        )
+        # Now, groups that contains those users
+        groups_list = set()
+        for u in self.output_count('Filtering groups', users_list):
+            groups_list.update(u.groups.all())
+
+        # now, export those authenticators
+        authenticators = [
+            authenticator_exporter(a)
+            for a in self.output_count('Saving authenticators', authenticators_list)
+        ]
+
+        # then, export those groups
+        groups = [
+            group_export(g) for g in self.output_count('Saving groups', groups_list)
+        ]
+
+        # finally, export users with the filter
+        users = [
+            user_exporter(u) for u in self.output_count('Saving users', users_list)
+        ]
+        return (
+            '# Authenticators\n'
+            + yaml.safe_dump(authenticators)
+            + '# Groups\n'
+            + yaml.safe_dump(groups)
+            + '# Users\n'
+            + yaml.safe_dump(users)
+        )
+
+    def export_groups(self, **kwargs: str) -> str:
+        """
+        Exports all groups to a list of dicts
+        """
+        # First export authenticators for groups with the filter
+        groups_list = list(
+            self.output_count(
+                'Filtering groups', self.apply_filter(models.Group, **kwargs)
+            )
+        )
+        authenticators_list = set(
+            [
+                g.manager
+                for g in self.output_count('Filtering authenticators', groups_list)
+            ]
+        )
+        authenticators = [
+            authenticator_exporter(a)
+            for a in self.output_count('Saving authenticators', authenticators_list)
+        ]
+
+        # then, export groups with the filter
+        groups = [
+            group_export(g) for g in self.output_count('Saving groups', groups_list)
+        ]
+
+        return (
+            '# Authenticators\n'
+            + yaml.safe_dump(authenticators)
+            + '# Groups\n'
+            + yaml.safe_dump(groups)
+        )
+
+    def export_networks(self, **kwargs: str) -> str:
+        """
+        Exports all networks to a list of dicts
+        """
+        return '# Networks\n' + yaml.safe_dump(
+            [
+                network_exporter(n)
+                for n in self.output_count(
+                    'Saving networks', self.apply_filter(models.Network, **kwargs)
+                )
+            ]
+        )
+
+    def export_transports(self, **kwargs: str) -> str:
+        """
+        Exports all transports to a list of dicts
+        """
+        # First, export networks for transports with the filter
+        transports_list = list(
+            self.output_count(
+                'Filtering transports', self.apply_filter(models.Transport, **kwargs)
+            )
+        )
+        networks_list = set()
+        for t in self.output_count('Filtering networks', transports_list):
+            networks_list.update(t.networks.all())
+        networks = [
+            network_exporter(n)
+            for n in self.output_count('Saving networks', networks_list)
+        ]
+
+        # then, export transports with the filter
+        transports = [
+            transport_exporter(t)
+            for t in self.output_count('Saving transports', transports_list)
+        ]
+
+        return (
+            '# Networks\n'
+            + yaml.safe_dump(networks)
+            + '# Transports\n'
+            + yaml.safe_dump(transports)
+        )
+
+    def export_osmanagers(self, **kwargs: str) -> str:
+        """
+        Exports all osmanagers to a list of dicts
+        """
+        return '# OSManagers\n' + yaml.safe_dump(
+            [
+                osmanager_exporter(o)
+                for o in self.output_count(
+                    'Saving osmanagers', self.apply_filter(models.OSManager, **kwargs)
+                )
+            ]
+        )
+
+    def remove_reduntant_entities(self, entities: typing.List[str]) -> typing.List[str]:
+        """
+        Removes redundant entities from the list
+        """
+        REPLACES: typing.Mapping[str, typing.List[str]] = {
+            'users': ['authenticators', 'groups'],
+            'groups': ['authenticators'],
+            'authenticators': [],
+            'transports': ['networks'],
+            'networks': [],
+            'osmanagers': [],
+            'services': ['providers'],
+            'providers': [],
+        }
+        entities = list(set(entities))  # remove duplicates
+        # Remove entities that are replaced by other entities
+        for entity in entities:
+            for replace in REPLACES.get(entity, []):
+                if replace in entities:
+                    entities.remove(replace)
+
+        return entities

server/src/uds/models/group.py

@@ -56,7 +56,7 @@ class Group(UUIDModel):
     This class represents a group, associated with one authenticator
     """
 
-    manager: 'models.ForeignKey[Group, Authenticator]' = UnsavedForeignKey(
+    manager: 'models.ForeignKey[Authenticator]' = UnsavedForeignKey(
         Authenticator, on_delete=models.CASCADE, related_name='groups'
     )
     name = models.CharField(max_length=128, db_index=True)

server/src/uds/models/managed_object_model.py

@@ -68,7 +68,7 @@ class ManagedObjectModel(UUIDModel):
         """
         Returns an environment valid for the record this object represents
         """
-        return Environment.getEnvForTableElement(self._meta.verbose_name, self.id)
+        return Environment.getEnvForTableElement(self._meta.verbose_name, self.id)  # type: ignore
 
     def deserialize(self, obj: Module, values: typing.Optional[typing.Dict[str, str]]):
         """

server/src/uds/models/service.py

@@ -61,12 +61,12 @@ class Service(ManagedObjectModel, TaggingMixin):  # type: ignore
     Server configuration).
     """
 
-    provider: 'models.ForeignKey[Service, Provider]' = models.ForeignKey(
+    provider: 'models.ForeignKey[Provider]' = models.ForeignKey(
         Provider, related_name='services', on_delete=models.CASCADE
     )
 
     # Proxy for this service
-    proxy: 'models.ForeignKey[Service, Proxy]' = models.ForeignKey(
+    proxy: 'models.ForeignKey[Proxy|None]' = models.ForeignKey(
         Proxy, null=True, blank=True, related_name='services', on_delete=models.CASCADE
     )
 

server/src/uds/models/user.py

@@ -69,7 +69,7 @@ class User(UUIDModel):
     password = models.CharField(
         max_length=128, default=''
     )  # Only used on "internal" sources or sources that "needs password"
-    mfaData = models.CharField(
+    mfa_data = models.CharField(
         max_length=128, default=''
     )  # Only used on "internal" sources
     staff_member = models.BooleanField(
@@ -84,7 +84,6 @@ class User(UUIDModel):
     objects: 'models.BaseManager[User]'
     groups: 'models.manager.RelatedManager[Group]'
     userServices: 'models.manager.RelatedManager[UserService]'
-    mfa: 'models.manager.RelatedManager[MFA]'
 
     class Meta(UUIDModel.Meta):
         """
@@ -224,8 +223,8 @@ class User(UUIDModel):
             If the key exists, the custom data will always contain something, but may be the values are the default ones.
 
         """
-        with storage.StorageAccess('manager' + self.manager.uuid) as store:
+        with storage.StorageAccess('manager' + str(self.manager.uuid)) as store:
-            return store[self.uuid + '_' + key]
+            return store[str(self.uuid) + '_' + key]
 
     def __str__(self):
         return 'User {} (id:{}) from auth {}'.format(
@@ -248,7 +247,7 @@ class User(UUIDModel):
         # be removed
         toDelete.getManager().removeUser(toDelete.name)
         # Remove related stored values
-        with storage.StorageAccess('manager' + toDelete.manager.uuid) as store:
+        with storage.StorageAccess('manager' + str(toDelete.manager.uuid)) as store:
             for key in store.keys():
                 store.delete(key)