diff --git a/server/src/server/settings.py.sample b/server/src/server/settings.py.sample index a2499de83..c0b8bfe9e 100644 --- a/server/src/server/settings.py.sample +++ b/server/src/server/settings.py.sample @@ -202,6 +202,12 @@ COMPRESS_PRECOMPILERS = ( ) if DEBUG: COMPRESS_DEBUG_TOGGLE = 'debug' +# +# Enable this if you need to allow round robin load balancing of web server +# This is so because we need to share the files between servers +# Another options is put /var/server/static on a shared nfs forder for all servers +# +# COMPRESS_STORAGE = 'uds.core.util.FileStorage.FileStorage' # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. diff --git a/server/src/uds/core/util/FileStorage.py b/server/src/uds/core/util/FileStorage.py new file mode 100644 index 000000000..f2e8ad9c5 --- /dev/null +++ b/server/src/uds/core/util/FileStorage.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +# +# Copyright (c) 2016 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' +@author: Adolfo Gómez, dkmaster at dkmon dot com +''' +# pylint: disable=no-name-in-module,import-error +from __future__ import unicode_literals + +from django.core.files import File +from django.core.files.storage import Storage +from uds.models.DBFile import DBFile +from django.conf import settings +from six.moves.urllib import parse as urlparse # @UnresolvedImport + +import six +import os +import logging + +logger = logging.getLogger(__name__) + + +class FileStorage(Storage): + def __init__(self, *args, **kwargs): + self._base_url = getattr(settings, 'FILE_STORAGE', '/files') + if self._base_url[-1] != '/': + self._base_url += '/' + + Storage.__init__(self, *args, **kwargs) + + def get_valid_name(self, name): + return name.replace('\\', os.path.sep) + + def _file(self, name): + return DBFile.objects.get(name=self.get_valid_name(name)) + + def _open(self, name, mode='rb'): + f = six.BytesIO(self._file(name).data) + f.name = name + f.mode = mode + return File(f) + + def _save(self, name, content): + name = self.get_valid_name(name) + try: + file = self._file(name) + except DBFile.DoesNotExist: + file = DBFile.objects.create(name=name) + + file.data = content.read() + file.save() + return name + + def accessed_time(self, name): + raise NotImplementedError + + def created_time(self, name): + return self._file(name).created + + def modified_time(self, name): + return self._file(name).modified + + def size(self, name): + return self._file(name).size + + def delete(self, name): + self._file(name).delete() + + def exists(self, name): + try: + self._file(name) + return True + except DBFile.DoesNotExist: + return False + + def url(self, name): + uuid = self._file(name).uuid + return urlparse.urljoin(self._base_url, uuid) diff --git a/server/src/uds/core/util/decorators.py b/server/src/uds/core/util/decorators.py index 1148afbda..0fc8e7b53 100644 --- a/server/src/uds/core/util/decorators.py +++ b/server/src/uds/core/util/decorators.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2012-2016 Virtual Cable S.L. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -39,7 +39,7 @@ from functools import wraps import logging -__updated__ = '2015-05-03' +__updated__ = '2016-04-05' logger = logging.getLogger(__name__) diff --git a/server/src/uds/migrations/0021_auto_20160401_0652.py b/server/src/uds/migrations/0021_auto_20160405_0429.py similarity index 82% rename from server/src/uds/migrations/0021_auto_20160401_0652.py rename to server/src/uds/migrations/0021_auto_20160405_0429.py index 18429a2e4..ca197d585 100644 --- a/server/src/uds/migrations/0021_auto_20160401_0652.py +++ b/server/src/uds/migrations/0021_auto_20160405_0429.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.9.2 on 2016-04-01 06:52 +# Generated by Django 1.9.5 on 2016-04-05 04:29 from __future__ import unicode_literals from django.db import migrations, models @@ -44,6 +44,20 @@ class Migration(migrations.Migration): 'db_table': 'uds_cal_action', }, ), + migrations.CreateModel( + name='DBFile', + fields=[ + ('uuid', models.CharField(default=None, max_length=50, null=True, unique=True)), + ('name', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('content', models.TextField(blank=True)), + ('size', models.IntegerField(default=0)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ], + options={ + 'abstract': False, + }, + ), migrations.AddField( model_name='deployedservice', name='fallbackAccess', diff --git a/server/src/uds/models/Calendar.py b/server/src/uds/models/Calendar.py index 30c12bd98..aa29674d3 100644 --- a/server/src/uds/models/Calendar.py +++ b/server/src/uds/models/Calendar.py @@ -2,7 +2,7 @@ # Model based on https://github.com/llazzaro/django-scheduler # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2016 Virtual Cable S.L. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -34,7 +34,7 @@ from __future__ import unicode_literals -__updated__ = '2016-02-17' +__updated__ = '2016-04-05' from django.db import models from uds.models.UUIDModel import UUIDModel diff --git a/server/src/uds/models/CalendarAccess.py b/server/src/uds/models/CalendarAccess.py index 8137c06ac..340641100 100644 --- a/server/src/uds/models/CalendarAccess.py +++ b/server/src/uds/models/CalendarAccess.py @@ -2,7 +2,7 @@ # Model based on https://github.com/llazzaro/django-scheduler # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2016 Virtual Cable S.L. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -34,7 +34,7 @@ from __future__ import unicode_literals -__updated__ = '2016-04-01' +__updated__ = '2016-04-05' from django.db import models from uds.core.util import states diff --git a/server/src/uds/models/CalendarAction.py b/server/src/uds/models/CalendarAction.py index ccd68f612..e574c5a49 100644 --- a/server/src/uds/models/CalendarAction.py +++ b/server/src/uds/models/CalendarAction.py @@ -2,7 +2,7 @@ # Model based on https://github.com/llazzaro/django-scheduler # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2016 Virtual Cable S.L. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -34,7 +34,7 @@ from __future__ import unicode_literals -__updated__ = '2016-04-01' +__updated__ = '2016-04-05' from django.utils.translation import ugettext_lazy as _ from django.db import models diff --git a/server/src/uds/models/CalendarRule.py b/server/src/uds/models/CalendarRule.py index 57d8e437e..302f30622 100644 --- a/server/src/uds/models/CalendarRule.py +++ b/server/src/uds/models/CalendarRule.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2016 Virtual Cable S.L. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -33,7 +33,7 @@ from __future__ import unicode_literals -__updated__ = '2016-03-14' +__updated__ = '2016-04-05' from django.db import models from django.utils.encoding import python_2_unicode_compatible diff --git a/server/src/uds/models/DBFile.py b/server/src/uds/models/DBFile.py new file mode 100644 index 000000000..c9fe86c4c --- /dev/null +++ b/server/src/uds/models/DBFile.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# Model based on https://github.com/llazzaro/django-scheduler +# +# Copyright (c) 2016 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' +.. moduleauthor:: Adolfo Gómez, dkmaster at dkmon dot com +''' + +from __future__ import unicode_literals + +__updated__ = '2016-04-05' + +from django.db import models +from django.utils.encoding import python_2_unicode_compatible +from uds.models.UUIDModel import UUIDModel + +import logging +import six + +logger = logging.getLogger(__name__) + + +@python_2_unicode_compatible +class DBFile(UUIDModel): + name = models.CharField(max_length=255, primary_key=True) + content = models.TextField(blank=True) + size = models.IntegerField(default=0) + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + @property + def data(self): + return self.content.decode('base64').decode('zip') + + @data.setter + def data(self, value): + self.size = len(value) + self.content = value.encode('zip').encode('base64') + + def __str__(self): + return 'File: {} {} {} {}'.format(self.name, self.size, self.created, self.modified) diff --git a/server/src/uds/models/Storage.py b/server/src/uds/models/Storage.py index 95c5f7829..8df9a9566 100644 --- a/server/src/uds/models/Storage.py +++ b/server/src/uds/models/Storage.py @@ -38,7 +38,7 @@ from uds.core.db.LockingManager import LockingManager import logging -__updated__ = '2015-06-01' +__updated__ = '2016-04-05' logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) class Storage(models.Model): ''' - General storage model. Used to store specific instances (transport, service, servicemanager, ...) persinstent information + General storage model. Used to store specific instances (transport, service, servicemanager, ...) persistent information not intended to be serialized/deserialized everytime one object instance is loaded/saved. ''' owner = models.CharField(max_length=128, db_index=True) diff --git a/server/src/uds/models/__init__.py b/server/src/uds/models/__init__.py index 12f3e7352..e557d7dca 100644 --- a/server/src/uds/models/__init__.py +++ b/server/src/uds/models/__init__.py @@ -105,7 +105,10 @@ from .CalendarAction import CalendarAction # Tagging from .Tag import Tag -__updated__ = '2016-02-17' +# Utility +from .DBFile import DBFile + +__updated__ = '2016-04-05' logger = logging.getLogger(__name__) diff --git a/server/src/uds/urls.py b/server/src/uds/urls.py index bd2cc8e41..c83a61d28 100644 --- a/server/src/uds/urls.py +++ b/server/src/uds/urls.py @@ -82,6 +82,9 @@ urlpatterns = patterns( # Web admin GUI (r'^adm/', include('uds.admin.urls')), + # Files + (r'^files/(?P.+)', 'web.views.file_storage'), + # Internacionalization in javascript # Javascript catalog (r'^jsi18n/(?P[a-z]*)$', 'web.views.jsCatalog', js_info_dict), diff --git a/server/src/uds/web/views/__init__.py b/server/src/uds/web/views/__init__.py index 2f0378398..717890631 100644 --- a/server/src/uds/web/views/__init__.py +++ b/server/src/uds/web/views/__init__.py @@ -42,7 +42,8 @@ from .client_download import client_downloads, plugin_detection from .js import jsCatalog from ..errors import error from .images import image +from .file_storage import file_storage -__updated__ = '2016-02-15' +__updated__ = '2016-04-05' logger = logging.getLogger(__name__) diff --git a/server/src/uds/web/views/client_download.py b/server/src/uds/web/views/client_download.py index e053d9100..5e9ae64f7 100644 --- a/server/src/uds/web/views/client_download.py +++ b/server/src/uds/web/views/client_download.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2012 Virtual Cable S.L. +# Copyright (c) 2012-2016 Virtual Cable S.L. # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -45,7 +45,7 @@ import logging logger = logging.getLogger(__name__) -__updated__ = '2015-04-28' +__updated__ = '2016-04-05' UserPrefsManager.manager().registerPrefs( diff --git a/server/src/uds/web/views/file_storage.py b/server/src/uds/web/views/file_storage.py new file mode 100644 index 000000000..559131332 --- /dev/null +++ b/server/src/uds/web/views/file_storage.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2012-2016 Virtual Cable S.L. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Virtual Cable S.L. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +''' +@author: Adolfo Gómez, dkmaster at dkmon dot com +''' +from __future__ import unicode_literals + +from uds.models import DBFile +from django.http import HttpResponse +from django.views.decorators.cache import cache_page + +import mimetypes +import logging + +logger = logging.getLogger(__name__) + +__updated__ = '2016-04-05' + +@cache_page(3600, key_prefix='file', cache='memory') +def file_storage(request, uuid): + f = DBFile.objects.get(uuid=uuid) + content_type, encoding = mimetypes.guess_type(f.name) + + return HttpResponse(f.data, content_type=content_type)