1
0
mirror of https://github.com/ansible/awx.git synced 2024-11-01 16:51:11 +03:00

Import mongoengine rest framework module, document this and mongoengine

in the readme
This commit is contained in:
Matthew Jones 2015-05-05 14:45:44 -04:00
parent f3739ec283
commit 1bd2c99171
10 changed files with 778 additions and 0 deletions

View File

@ -38,6 +38,8 @@ keystoneclient==1.3.0 (keystone/*)
kombu==3.0.21 (kombu/*) kombu==3.0.21 (kombu/*)
Markdown==2.4.1 (markdown/*, excluded bin/markdown_py) Markdown==2.4.1 (markdown/*, excluded bin/markdown_py)
mock==1.0.1 (mock.py) mock==1.0.1 (mock.py)
mongoengine==0.9.0 (mongoengine/*)
mongoengine_rest_framework==1.5.4 (rest_framework_mongoengine/*)
netaddr==0.7.14 (netaddr/*) netaddr==0.7.14 (netaddr/*)
os_client_config==0.6.0 (os_client_config/*) os_client_config==0.6.0 (os_client_config/*)
ordereddict==1.1 (ordereddict.py, needed for Python 2.6 support) ordereddict==1.1 (ordereddict.py, needed for Python 2.6 support)

View File

@ -0,0 +1,137 @@
from bson.errors import InvalidId
from django.core.exceptions import ValidationError
from django.utils.encoding import smart_str
from mongoengine import dereference
from mongoengine.base.document import BaseDocument
from mongoengine.document import Document
from rest_framework import serializers
from mongoengine.fields import ObjectId
import bson
class MongoDocumentField(serializers.WritableField):
MAX_RECURSION_DEPTH = 5 # default value of depth
def __init__(self, *args, **kwargs):
try:
self.model_field = kwargs.pop('model_field')
self.depth = kwargs.pop('depth', self.MAX_RECURSION_DEPTH)
except KeyError:
raise ValueError("%s requires 'model_field' kwarg" % self.type_label)
super(MongoDocumentField, self).__init__(*args, **kwargs)
def transform_document(self, document, depth):
data = {}
# serialize each required field
for field in document._fields:
if hasattr(document, smart_str(field)):
# finally check for an attribute 'field' on the instance
obj = getattr(document, field)
else:
continue
val = self.transform_object(obj, depth-1)
if val is not None:
data[field] = val
return data
def transform_dict(self, obj, depth):
return dict([(key, self.transform_object(val, depth-1))
for key, val in obj.items()])
def transform_object(self, obj, depth):
"""
Models to natives
Recursion for (embedded) objects
"""
if isinstance(obj, BaseDocument):
# Document, EmbeddedDocument
if depth == 0:
# Return primary key if exists, else return default text
return smart_str(getattr(obj, 'pk', 'Max recursion depth exceeded'))
return self.transform_document(obj, depth)
elif isinstance(obj, dict):
# Dictionaries
return self.transform_dict(obj, depth)
elif isinstance(obj, list):
# List
return [self.transform_object(value, depth) for value in obj]
elif obj is None:
return None
else:
return smart_str(obj) if isinstance(obj, ObjectId) else obj
class ReferenceField(MongoDocumentField):
type_label = 'ReferenceField'
def from_native(self, value):
try:
dbref = self.model_field.to_python(value)
except InvalidId:
raise ValidationError(self.error_messages['invalid'])
instance = dereference.DeReference().__call__([dbref])[0]
# Check if dereference was successful
if not isinstance(instance, Document):
msg = self.error_messages['invalid']
raise ValidationError(msg)
return instance
def to_native(self, obj):
#if type is DBRef it means Mongo can't find the actual reference object
#prevent the JSON serializable error by setting the object to None
if type(obj) == bson.dbref.DBRef:
obj = None
return self.transform_object(obj, self.depth - 1)
class ListField(MongoDocumentField):
type_label = 'ListField'
def from_native(self, value):
return self.model_field.to_python(value)
def to_native(self, obj):
return self.transform_object(obj, self.depth - 1)
class EmbeddedDocumentField(MongoDocumentField):
type_label = 'EmbeddedDocumentField'
def __init__(self, *args, **kwargs):
try:
self.document_type = kwargs.pop('document_type')
except KeyError:
raise ValueError("EmbeddedDocumentField requires 'document_type' kwarg")
super(EmbeddedDocumentField, self).__init__(*args, **kwargs)
def get_default_value(self):
return self.to_native(self.default())
def to_native(self, obj):
if obj is None:
return None
else:
return self.transform_object(obj, self.depth)
def from_native(self, value):
return self.model_field.to_python(value)
class DynamicField(MongoDocumentField):
type_label = 'DynamicField'
def to_native(self, obj):
return self.model_field.to_python(obj)

View File

@ -0,0 +1,150 @@
from django.core.exceptions import ImproperlyConfigured
from rest_framework import mixins
from rest_framework.generics import GenericAPIView
from mongoengine.django.shortcuts import get_document_or_404
class MongoAPIView(GenericAPIView):
"""
Mixin for views manipulating mongo documents
"""
queryset = None
serializer_class = None
lookup_field = 'id'
def get_queryset(self):
"""
Get the list of items for this view.
This must be an iterable, and may be a queryset.
Defaults to using `self.queryset`.
You may want to override this if you need to provide different
querysets depending on the incoming request.
(Eg. return a list of items that is specific to the user)
"""
if self.queryset is not None:
return self.queryset.clone()
if self.model is not None:
return self.get_serializer().opts.model.objects.all()
raise ImproperlyConfigured("'%s' must define 'queryset' or 'model'"
% self.__class__.__name__)
def get_object(self, queryset=None):
"""
Get a document instance for read/update/delete requests.
"""
query_key = self.lookup_url_kwarg or self.lookup_field
query_kwargs = {query_key: self.kwargs[query_key]}
queryset = self.get_queryset()
obj = get_document_or_404(queryset, **query_kwargs)
self.check_object_permissions(self.request, obj)
return obj
class CreateAPIView(mixins.CreateModelMixin,
MongoAPIView):
"""
Concrete view for creating a model instance.
"""
def post(self, request, *args, **kwargs):
return self.create(request, *args, **kwargs)
class ListAPIView(mixins.ListModelMixin,
MongoAPIView):
"""
Concrete view for listing a queryset.
"""
def get(self, request, *args, **kwargs):
return self.list(request, *args, **kwargs)
class ListCreateAPIView(mixins.ListModelMixin,
mixins.CreateModelMixin,
MongoAPIView):
"""
Concrete view for listing a queryset or creating a model instance.
"""
def get(self, request, *args, **kwargs):
return self.list(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
return self.create(request, *args, **kwargs)
class RetrieveAPIView(mixins.RetrieveModelMixin,
MongoAPIView):
"""
Concrete view for retrieving a model instance.
"""
def get(self, request, *args, **kwargs):
return self.retrieve(request, *args, **kwargs)
class UpdateAPIView(mixins.UpdateModelMixin,
MongoAPIView):
"""
Concrete view for updating a model instance.
"""
def put(self, request, *args, **kwargs):
return self.update(request, *args, **kwargs)
def patch(self, request, *args, **kwargs):
return self.partial_update(request, *args, **kwargs)
class RetrieveUpdateAPIView(mixins.RetrieveModelMixin,
mixins.UpdateModelMixin,
MongoAPIView):
"""
Concrete view for retrieving, updating a model instance.
"""
def get(self, request, *args, **kwargs):
return self.retrieve(request, *args, **kwargs)
def put(self, request, *args, **kwargs):
return self.update(request, *args, **kwargs)
def patch(self, request, *args, **kwargs):
return self.partial_update(request, *args, **kwargs)
class RetrieveDestroyAPIView(mixins.RetrieveModelMixin,
mixins.DestroyModelMixin,
MongoAPIView):
"""
Concrete view for retrieving or deleting a model instance.
"""
def get(self, request, *args, **kwargs):
return self.retrieve(request, *args, **kwargs)
def delete(self, request, *args, **kwargs):
return self.destroy(request, *args, **kwargs)
class RetrieveUpdateDestroyAPIView(mixins.RetrieveModelMixin,
mixins.UpdateModelMixin,
mixins.DestroyModelMixin,
MongoAPIView):
"""
Concrete view for retrieving, updating or deleting a model instance.
"""
def get(self, request, *args, **kwargs):
return self.retrieve(request, *args, **kwargs)
def put(self, request, *args, **kwargs):
return self.update(request, *args, **kwargs)
def patch(self, request, *args, **kwargs):
return self.partial_update(request, *args, **kwargs)
def delete(self, request, *args, **kwargs):
return self.destroy(request, *args, **kwargs)

View File

@ -0,0 +1,22 @@
from rest_framework.routers import SimpleRouter, DefaultRouter
class MongoRouterMixin(object):
def get_default_base_name(self, viewset):
"""
If `base_name` is not specified, attempt to automatically determine
it from the viewset.
"""
model_cls = getattr(viewset, 'model', None)
assert model_cls, '`base_name` argument not specified, and could ' \
'not automatically determine the name from the viewset, as ' \
'it does not have a `.model` attribute.'
return model_cls.__name__.lower()
class MongoSimpleRouter(MongoRouterMixin, SimpleRouter):
pass
class MongoDefaultRouter(MongoSimpleRouter, DefaultRouter):
pass

View File

@ -0,0 +1,268 @@
from __future__ import unicode_literals
import warnings
from mongoengine.errors import ValidationError
from rest_framework import serializers
from rest_framework import fields
import mongoengine
from mongoengine.base import BaseDocument
from django.core.paginator import Page
from django.db import models
from django.forms import widgets
from django.utils.datastructures import SortedDict
from rest_framework.compat import get_concrete_model
from .fields import ReferenceField, ListField, EmbeddedDocumentField, DynamicField
class MongoEngineModelSerializerOptions(serializers.ModelSerializerOptions):
"""
Meta class options for MongoEngineModelSerializer
"""
def __init__(self, meta):
super(MongoEngineModelSerializerOptions, self).__init__(meta)
self.depth = getattr(meta, 'depth', 5)
class MongoEngineModelSerializer(serializers.ModelSerializer):
"""
Model Serializer that supports Mongoengine
"""
_options_class = MongoEngineModelSerializerOptions
def perform_validation(self, attrs):
"""
Rest Framework built-in validation + related model validations
"""
for field_name, field in self.fields.items():
if field_name in self._errors:
continue
source = field.source or field_name
if self.partial and source not in attrs:
continue
if field_name in attrs and hasattr(field, 'model_field'):
try:
field.model_field.validate(attrs[field_name])
except ValidationError as err:
self._errors[field_name] = str(err)
try:
validate_method = getattr(self, 'validate_%s' % field_name, None)
if validate_method:
attrs = validate_method(attrs, source)
except serializers.ValidationError as err:
self._errors[field_name] = self._errors.get(field_name, []) + list(err.messages)
if not self._errors:
try:
attrs = self.validate(attrs)
except serializers.ValidationError as err:
if hasattr(err, 'message_dict'):
for field_name, error_messages in err.message_dict.items():
self._errors[field_name] = self._errors.get(field_name, []) + list(error_messages)
elif hasattr(err, 'messages'):
self._errors['non_field_errors'] = err.messages
return attrs
def restore_object(self, attrs, instance=None):
if instance is None:
instance = self.opts.model()
dynamic_fields = self.get_dynamic_fields(instance)
all_fields = dict(dynamic_fields, **self.fields)
for key, val in attrs.items():
field = all_fields.get(key)
if not field or field.read_only:
continue
if isinstance(field, serializers.Serializer):
many = field.many
def _restore(field, item):
# looks like a bug, sometimes there are decerialized objects in attrs
# sometimes they are just dicts
if isinstance(item, BaseDocument):
return item
return field.from_native(item)
if many:
val = [_restore(field, item) for item in val]
else:
val = _restore(field, val)
key = getattr(field, 'source', None) or key
try:
setattr(instance, key, val)
except ValueError:
self._errors[key] = self.error_messages['required']
return instance
def get_default_fields(self):
cls = self.opts.model
opts = get_concrete_model(cls)
fields = []
fields += [getattr(opts, field) for field in cls._fields_ordered]
ret = SortedDict()
for model_field in fields:
if isinstance(model_field, mongoengine.ObjectIdField):
field = self.get_pk_field(model_field)
else:
field = self.get_field(model_field)
if field:
field.initialize(parent=self, field_name=model_field.name)
ret[model_field.name] = field
for field_name in self.opts.read_only_fields:
assert field_name in ret,\
"read_only_fields on '%s' included invalid item '%s'" %\
(self.__class__.__name__, field_name)
ret[field_name].read_only = True
for field_name in self.opts.write_only_fields:
assert field_name in ret,\
"write_only_fields on '%s' included invalid item '%s'" %\
(self.__class__.__name__, field_name)
ret[field_name].write_only = True
return ret
def get_dynamic_fields(self, obj):
dynamic_fields = {}
if obj is not None and obj._dynamic:
for key, value in obj._dynamic_fields.items():
dynamic_fields[key] = self.get_field(value)
return dynamic_fields
def get_field(self, model_field):
kwargs = {}
if model_field.__class__ in (mongoengine.ReferenceField, mongoengine.EmbeddedDocumentField,
mongoengine.ListField, mongoengine.DynamicField):
kwargs['model_field'] = model_field
kwargs['depth'] = self.opts.depth
if not model_field.__class__ == mongoengine.ObjectIdField:
kwargs['required'] = model_field.required
if model_field.__class__ == mongoengine.EmbeddedDocumentField:
kwargs['document_type'] = model_field.document_type
if model_field.default:
kwargs['required'] = False
kwargs['default'] = model_field.default
if model_field.__class__ == models.TextField:
kwargs['widget'] = widgets.Textarea
field_mapping = {
mongoengine.FloatField: fields.FloatField,
mongoengine.IntField: fields.IntegerField,
mongoengine.DateTimeField: fields.DateTimeField,
mongoengine.EmailField: fields.EmailField,
mongoengine.URLField: fields.URLField,
mongoengine.StringField: fields.CharField,
mongoengine.BooleanField: fields.BooleanField,
mongoengine.FileField: fields.FileField,
mongoengine.ImageField: fields.ImageField,
mongoengine.ObjectIdField: fields.WritableField,
mongoengine.ReferenceField: ReferenceField,
mongoengine.ListField: ListField,
mongoengine.EmbeddedDocumentField: EmbeddedDocumentField,
mongoengine.DynamicField: DynamicField,
mongoengine.DecimalField: fields.DecimalField,
mongoengine.UUIDField: fields.CharField
}
attribute_dict = {
mongoengine.StringField: ['max_length'],
mongoengine.DecimalField: ['min_value', 'max_value'],
mongoengine.EmailField: ['max_length'],
mongoengine.FileField: ['max_length'],
mongoengine.URLField: ['max_length'],
}
if model_field.__class__ in attribute_dict:
attributes = attribute_dict[model_field.__class__]
for attribute in attributes:
kwargs.update({attribute: getattr(model_field, attribute)})
try:
return field_mapping[model_field.__class__](**kwargs)
except KeyError:
# Defaults to WritableField if not in field mapping
return fields.WritableField(**kwargs)
def to_native(self, obj):
"""
Rest framework built-in to_native + transform_object
"""
ret = self._dict_class()
ret.fields = self._dict_class()
#Dynamic Document Support
dynamic_fields = self.get_dynamic_fields(obj)
all_fields = self._dict_class()
all_fields.update(self.fields)
all_fields.update(dynamic_fields)
for field_name, field in all_fields.items():
if field.read_only and obj is None:
continue
field.initialize(parent=self, field_name=field_name)
key = self.get_field_key(field_name)
value = field.field_to_native(obj, field_name)
#Override value with transform_ methods
method = getattr(self, 'transform_%s' % field_name, None)
if callable(method):
value = method(obj, value)
if not getattr(field, 'write_only', False):
ret[key] = value
ret.fields[key] = self.augment_field(field, field_name, key, value)
return ret
def from_native(self, data, files=None):
self._errors = {}
if data is not None or files is not None:
attrs = self.restore_fields(data, files)
for key in data.keys():
if key not in attrs:
attrs[key] = data[key]
if attrs is not None:
attrs = self.perform_validation(attrs)
else:
self._errors['non_field_errors'] = ['No input provided']
if not self._errors:
return self.restore_object(attrs, instance=getattr(self, 'object', None))
@property
def data(self):
"""
Returns the serialized data on the serializer.
"""
if self._data is None:
obj = self.object
if self.many is not None:
many = self.many
else:
many = hasattr(obj, '__iter__') and not isinstance(obj, (BaseDocument, Page, dict))
if many:
warnings.warn('Implicit list/queryset serialization is deprecated. '
'Use the `many=True` flag when instantiating the serializer.',
DeprecationWarning, stacklevel=2)
if many:
self._data = [self.to_native(item) for item in obj]
else:
self._data = self.to_native(obj)
return self._data

View File

@ -0,0 +1,6 @@
import os
import sys
sys.path.insert(0, os.path.abspath('../'))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Sample.settings")

View File

@ -0,0 +1,7 @@
Django==1.6.5
argparse==1.2.1
djangorestframework==2.3.14
mongoengine==0.8.7
nose==1.3.3
pymongo==2.7.1
wsgiref==0.1.2

View File

@ -0,0 +1,152 @@
from datetime import datetime
import mongoengine as me
from unittest import TestCase
from bson import objectid
from rest_framework_mongoengine.serializers import MongoEngineModelSerializer
from rest_framework import serializers as s
class Job(me.Document):
title = me.StringField()
status = me.StringField(choices=('draft', 'published'))
notes = me.StringField(required=False)
on = me.DateTimeField(default=datetime.utcnow)
weight = me.IntField(default=0)
class JobSerializer(MongoEngineModelSerializer):
id = s.Field()
title = s.CharField()
status = s.ChoiceField(read_only=True)
sort_weight = s.IntegerField(source='weight')
class Meta:
model = Job
fields = ('id', 'title','status', 'sort_weight')
class TestReadonlyRestore(TestCase):
def test_restore_object(self):
job = Job(title='original title', status='draft', notes='secure')
data = {
'title': 'updated title ...',
'status': 'published', # this one is read only
'notes': 'hacked', # this field should not update
'sort_weight': 10 # mapped to a field with differet name
}
serializer = JobSerializer(job, data=data, partial=True)
self.assertTrue(serializer.is_valid())
obj = serializer.object
self.assertEqual(data['title'], obj.title)
self.assertEqual('draft', obj.status)
self.assertEqual('secure', obj.notes)
self.assertEqual(10, obj.weight)
# Testing restoring embedded property
class Location(me.EmbeddedDocument):
city = me.StringField()
# list of
class Category(me.EmbeddedDocument):
id = me.StringField()
counter = me.IntField(default=0, required=True)
class Secret(me.EmbeddedDocument):
key = me.StringField()
class SomeObject(me.Document):
name = me.StringField()
loc = me.EmbeddedDocumentField('Location')
categories = me.ListField(me.EmbeddedDocumentField(Category))
codes = me.ListField(me.EmbeddedDocumentField(Secret))
class LocationSerializer(MongoEngineModelSerializer):
city = s.CharField()
class Meta:
model = Location
class CategorySerializer(MongoEngineModelSerializer):
id = s.CharField(max_length=24)
class Meta:
model = Category
fields = ('id',)
class SomeObjectSerializer(MongoEngineModelSerializer):
location = LocationSerializer(source='loc')
categories = CategorySerializer(many=True, allow_add_remove=True)
class Meta:
model = SomeObject
fields = ('name', 'location', 'categories')
class TestRestoreEmbedded(TestCase):
def setUp(self):
self.data = {
'name': 'some anme',
'location': {
'city': 'Toronto'
},
'categories': [{'id': 'cat1'}, {'id': 'category_2', 'counter': 666}],
'codes': [{'key': 'mykey1'}]
}
def test_restore_new(self):
serializer = SomeObjectSerializer(data=self.data)
self.assertTrue(serializer.is_valid())
obj = serializer.object
self.assertEqual(self.data['name'], obj.name )
self.assertEqual('Toronto', obj.loc.city )
self.assertEqual(2, len(obj.categories))
self.assertEqual('category_2', obj.categories[1].id)
# counter is not listed in serializer fields, cannot be updated
self.assertEqual(0, obj.categories[1].counter)
# codes are not listed, should not be updatable
self.assertEqual(0, len(obj.codes))
def test_restore_update(self):
data = self.data
instance = SomeObject(
name='original',
loc=Location(city="New York"),
categories=[Category(id='orig1', counter=777)],
codes=[Secret(key='confidential123')]
)
serializer = SomeObjectSerializer(instance, data=data, partial=True)
# self.assertTrue(serializer.is_valid())
if not serializer.is_valid():
print 'errors: %s' % serializer._errors
assert False, 'errors'
obj = serializer.object
self.assertEqual(data['name'], obj.name )
self.assertEqual('Toronto', obj.loc.city )
# codes is not listed, should not be updatable
self.assertEqual(1, len(obj.codes[0]))
self.assertEqual('confidential123', obj.codes[0].key) # should keep original val
self.assertEqual(2, len(obj.categories))
self.assertEqual('category_2', obj.categories[1].id)
self.assertEqual(0, obj.categories[1].counter)

View File

@ -0,0 +1,34 @@
from rest_framework import mixins
from rest_framework.viewsets import ViewSetMixin
from rest_framework_mongoengine.generics import MongoAPIView
class MongoGenericViewSet(ViewSetMixin, MongoAPIView):
"""
The MongoGenericViewSet class does not provide any actions by default,
but does include the base set of generic view behavior, such as
the `get_object` and `get_queryset` methods.
"""
pass
class ModelViewSet(mixins.CreateModelMixin,
mixins.RetrieveModelMixin,
mixins.UpdateModelMixin,
mixins.DestroyModelMixin,
mixins.ListModelMixin,
MongoGenericViewSet):
"""
A viewset that provides default `create()`, `retrieve()`, `update()`,
`partial_update()`, `destroy()` and `list()` actions.
"""
pass
class ReadOnlyModelViewSet(mixins.RetrieveModelMixin,
mixins.ListModelMixin,
MongoGenericViewSet):
"""
A viewset that provides default `list()` and `retrieve()` actions.
"""
pass