Merge pull request #180 from chrismeyersfsu/explore-run_fact_cache_receiver_update

fixes Fact serialization/deserialization

awx/fact/models/fact.py

@@ -1,7 +1,24 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved
 
-from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField
+from mongoengine.base import BaseField
+from mongoengine import Document, DateTimeField, ReferenceField, StringField
+from awx.fact.utils.dbtransform import KeyTransform
+
+key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')])
+
+class TransformField(BaseField):
+    def to_python(self, value):
+        return key_transform.transform_outgoing(value, None)
+
+    def prepare_query_value(self, op, value):
+        if op == 'set':
+            value = key_transform.transform_incoming(value, None)
+        return super(TransformField, self).prepare_query_value(op, value)
+
+    def to_mongo(self, value):
+        value = key_transform.transform_incoming(value, None)
+        return value
 
 class FactHost(Document):
     hostname = StringField(max_length=100, required=True, unique=True)
@@ -21,11 +38,11 @@ class FactHost(Document):
             return host.id
         return None
 
-class Fact(DynamicDocument):
+class Fact(Document):
     timestamp = DateTimeField(required=True)
     host = ReferenceField(FactHost, required=True)
     module = StringField(max_length=50, required=True)
-    # fact = <anything>
+    fact = TransformField(required=True)
 
     # TODO: Consider using hashed index on host. django-mongo may not support this but
     # executing raw js will

awx/fact/tests/models/fact/__init__.py

@@ -4,4 +4,6 @@
 from __future__ import absolute_import
 
 from .fact_simple import * # noqa
+from .fact_transform_pymongo import * # noqa
+from .fact_transform import * # noqa
 from .fact_get_single_facts import * # noqa

awx/fact/tests/models/fact/fact_transform.py

@@ -0,0 +1,112 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+# Python
+from __future__ import absolute_import
+from datetime import datetime
+
+# Django
+from django.conf import settings
+
+# Pymongo
+import pymongo
+
+# AWX
+from awx.fact.models.fact import * # noqa
+from .base import BaseFactTest
+
+__all__ = ['FactTransformTest', 'FactTransformUpdateTest',]
+
+TEST_FACT_DATA = {
+    'hostname': 'hostname1',
+    'add_fact_data': {
+        'timestamp': datetime.now(),
+        'host': None,
+        'module': 'packages',
+        'fact': {
+            "acpid3.4": [
+                {
+                    "version": "1:2.0.21-1ubuntu2",
+                    "deeper.key": "some_value"
+                }
+            ],
+            "adduser.2": [
+                {
+                    "source": "apt",
+                    "version": "3.113+nmu3ubuntu3"
+                }
+            ],
+            "what.ever." : {
+                "shallowish.key": "some_shallow_value"
+            }
+        },
+    }
+}
+# Strip off microseconds because mongo has less precision
+BaseFactTest.normalize_timestamp(TEST_FACT_DATA)
+
+class FactTransformTest(BaseFactTest):
+    def setUp(self):
+        super(FactTransformTest, self).setUp()
+        # TODO: get host settings from config
+        self.client = pymongo.MongoClient('localhost', 27017)
+        self.db2 = self.client[settings.MONGO_DB]
+
+        self.create_host_document(TEST_FACT_DATA)
+
+    def setup_create_fact_dot(self):
+        self.data = TEST_FACT_DATA
+        self.f = Fact(**TEST_FACT_DATA['add_fact_data'])
+        self.f.save()
+
+    def setup_create_fact_dollar(self):
+        self.data = TEST_FACT_DATA
+        self.f = Fact(**TEST_FACT_DATA['add_fact_data'])
+        self.f.save()
+
+    def test_fact_with_dot_serialized(self):
+        self.setup_create_fact_dot()
+
+        q = {
+            '_id': self.f.id
+        }
+
+        # Bypass mongoengine and pymongo transform to get record
+        f_dict = self.db2['fact'].find_one(q)
+        self.assertIn('acpid3\uff0E4', f_dict['fact'])
+
+    def test_fact_with_dot_serialized_pymongo(self):
+        #self.setup_create_fact_dot()
+
+        f = self.db['fact'].insert({ 
+            'hostname': TEST_FACT_DATA['hostname'],
+            'fact': TEST_FACT_DATA['add_fact_data']['fact'],
+            'timestamp': TEST_FACT_DATA['add_fact_data']['timestamp'],
+            'host': TEST_FACT_DATA['add_fact_data']['host'].id,
+            'module': TEST_FACT_DATA['add_fact_data']['module']
+        })
+
+        q = {
+            '_id': f
+        }
+        # Bypass mongoengine and pymongo transform to get record
+        f_dict = self.db2['fact'].find_one(q)
+        self.assertIn('acpid3\uff0E4', f_dict['fact'])
+
+    def test_fact_with_dot_deserialized_pymongo(self):
+        self.setup_create_fact_dot()
+
+        q = {
+            '_id': self.f.id
+        }
+        f_dict = self.db['fact'].find_one(q)
+        self.assertIn('acpid3.4', f_dict['fact'])
+
+    def test_fact_with_dot_deserialized(self):
+        self.setup_create_fact_dot()
+
+        f = Fact.objects.get(id=self.f.id)
+        self.assertIn('acpid3.4', f.fact)
+
+class FactTransformUpdateTest(BaseFactTest):
+    pass

awx/fact/tests/models/fact/fact_transform_pymongo.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+# Python
+from __future__ import absolute_import
+from datetime import datetime
+
+# Django
+from django.conf import settings
+
+# Pymongo
+import pymongo
+
+# AWX
+from awx.fact.models.fact import * # noqa
+from .base import BaseFactTest
+
+__all__ = ['FactSerializePymongoTest', 'FactDeserializePymongoTest',]
+
+class FactPymongoBaseTest(BaseFactTest):
+    def setUp(self):
+        super(FactPymongoBaseTest, self).setUp()
+        # TODO: get host settings from config
+        self.client = pymongo.MongoClient('localhost', 27017)
+        self.db2 = self.client[settings.MONGO_DB]
+
+    def _create_fact(self):
+        fact = {}
+        fact[self.k] = self.v
+        q = {
+            'hostname': 'blah'
+        }
+        h = self.db['fact_host'].insert(q)
+        q = { 
+            'host': h,
+            'module': 'blah',
+            'timestamp': datetime.now(),
+            'fact': fact
+        }
+        f = self.db['fact'].insert(q)
+        return f
+
+    def check_transform(self, id):
+        raise RuntimeError("Must override")
+
+    def create_dot_fact(self):
+        self.k = 'this.is.a.key'
+        self.v = 'this.is.a.value'
+
+        self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey'
+
+        return self._create_fact()
+
+    def create_dollar_fact(self):
+        self.k = 'this$is$a$key'
+        self.v = 'this$is$a$value'
+
+        self.k_uni = 'this\uff04is\uff04a\uff04key'
+
+        return self._create_fact()
+
+class FactSerializePymongoTest(FactPymongoBaseTest):
+    def check_transform(self, id):
+        q = {
+            '_id': id
+        }
+        f = self.db2.fact.find_one(q)
+        self.assertIn(self.k_uni, f['fact'])
+        self.assertEqual(f['fact'][self.k_uni], self.v)
+
+    # Ensure key . are being transformed to the equivalent unicode into the database
+    def test_key_transform_dot(self):
+        f = self.create_dot_fact()
+        self.check_transform(f)
+
+    # Ensure key $ are being transformed to the equivalent unicode into the database
+    def test_key_transform_dollar(self):
+        f = self.create_dollar_fact()
+        self.check_transform(f)
+
+class FactDeserializePymongoTest(FactPymongoBaseTest):
+    def check_transform(self, id):
+        q = {
+            '_id': id
+        }
+        f = self.db.fact.find_one(q)
+        self.assertIn(self.k, f['fact'])
+        self.assertEqual(f['fact'][self.k], self.v)
+
+    def test_key_transform_dot(self):
+        f = self.create_dot_fact()
+        self.check_transform(f)
+
+    def test_key_transform_dollar(self):
+        f = self.create_dollar_fact()
+        self.check_transform(f)

awx/fact/tests/utils/dbtransform.py

@@ -1,77 +1,112 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved
 
-# Python
-from datetime import datetime
-from mongoengine import connect
-
-# Django
-from django.conf import settings
-
 # AWX
-from awx.main.tests.base import BaseTest, MongoDBRequired
+from awx.main.tests.base import BaseTest
 from awx.fact.models.fact import * # noqa
+from awx.fact.utils.dbtransform import KeyTransform
 
-__all__ = ['DBTransformTest']
+#__all__ = ['DBTransformTest', 'KeyTransformUnitTest']
+__all__ = ['KeyTransformUnitTest']
 
-class DBTransformTest(BaseTest, MongoDBRequired):
+class KeyTransformUnitTest(BaseTest):
     def setUp(self):
-        super(DBTransformTest, self).setUp()
+        super(KeyTransformUnitTest, self).setUp()
+        self.key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')])
 
-        # Create a db connection that doesn't have the transformation registered
-        # Note: this goes through pymongo not mongoengine
-        self.client = connect(settings.MONGO_DB)
-        self.db = self.client[settings.MONGO_DB]
+    def test_no_replace(self):
+        value = {
+            "a_key_with_a_dict" : {
+                "key" : "value",
+                "nested_key_with_dict": {
+                    "nested_key_with_value" : "deep_value"
+                }
+            }
+        }
 
-    def _create_fact(self):
-        fact = {}
-        fact[self.k] = self.v
-        h = FactHost(hostname='blah')
-        h.save()
-        f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact)
-        f.save()
-        return f
+        data = self.key_transform.transform_incoming(value, None)
+        self.assertEqual(data, value)
 
-    def create_dot_fact(self):
-        self.k = 'this.is.a.key'
-        self.v = 'this.is.a.value'
+        data = self.key_transform.transform_outgoing(value, None)
+        self.assertEqual(data, value)
 
-        self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey'
+    def test_complex(self):
+        value = {
+            "a.key.with.a.dict" : {
+                "key" : "value",
+                "nested.key.with.dict": {
+                    "nested.key.with.value" : "deep_value"
+                }
+            }
+        }
+        value_transformed = {
+            "a\uff0Ekey\uff0Ewith\uff0Ea\uff0Edict" : {
+                "key" : "value",
+                "nested\uff0Ekey\uff0Ewith\uff0Edict": {
+                    "nested\uff0Ekey\uff0Ewith\uff0Evalue" : "deep_value"
+                }
+            }
+        }
 
-        return self._create_fact()
+        data = self.key_transform.transform_incoming(value, None)
+        self.assertEqual(data, value_transformed)
 
-    def create_dollar_fact(self):
-        self.k = 'this$is$a$key'
-        self.v = 'this$is$a$value'
+        data = self.key_transform.transform_outgoing(value_transformed, None)
+        self.assertEqual(data, value)
 
-        self.k_uni = 'this\uff04is\uff04a\uff04key'
+    def test_simple(self):
+        value = {
+            "a.key" : "value"
+        }
+        value_transformed = {
+            "a\uff0Ekey" : "value"
+        }
 
-        return self._create_fact()
+        data = self.key_transform.transform_incoming(value, None)
+        self.assertEqual(data, value_transformed)
 
-    def check_unicode(self, f):
-        f_raw = self.db.fact.find_one(id=f.id)
-        self.assertIn(self.k_uni, f_raw['fact'])
-        self.assertEqual(f_raw['fact'][self.k_uni], self.v)
+        data = self.key_transform.transform_outgoing(value_transformed, None)
+        self.assertEqual(data, value)
 
-    # Ensure key . are being transformed to the equivalent unicode into the database
-    def test_key_transform_dot_unicode_in_storage(self):
-        f = self.create_dot_fact()
-        self.check_unicode(f)
+    def test_nested_dict(self):
+        value = {
+            "a.key.with.a.dict" : {
+                "nested.key." : "value"
+            }
+        }
+        value_transformed = {
+            "a\uff0Ekey\uff0Ewith\uff0Ea\uff0Edict" : {
+                "nested\uff0Ekey\uff0E" : "value"
+            }
+        }
 
-    # Ensure key $ are being transformed to the equivalent unicode into the database
-    def test_key_transform_dollar_unicode_in_storage(self):
-        f = self.create_dollar_fact()
-        self.check_unicode(f)
+        data = self.key_transform.transform_incoming(value, None)
+        self.assertEqual(data, value_transformed)
+        
+        data = self.key_transform.transform_outgoing(value_transformed, None)
+        self.assertEqual(data, value)
 
-    def check_transform(self):
-        f = Fact.objects.all()[0]
-        self.assertIn(self.k, f.fact)
-        self.assertEqual(f.fact[self.k], self.v)
+    def test_array(self):
+        value = {
+            "a.key.with.an.array" : [
+                {
+                    "key.with.dot" : "value"
+                }
+            ]
+        }
+        value_transformed = {
+            "a\uff0Ekey\uff0Ewith\uff0Ean\uff0Earray" : [
+                {
+                    "key\uff0Ewith\uff0Edot" : "value"
+                }
+            ]
+        }
+        data = self.key_transform.transform_incoming(value, None)
+        self.assertEqual(data, value_transformed)
+        
+        data = self.key_transform.transform_outgoing(value_transformed, None)
+        self.assertEqual(data, value)
 
-    def test_key_transform_dot_on_retreive(self):
-        self.create_dot_fact()
-        self.check_transform()
-
-    def test_key_transform_dollar_on_retreive(self):
-        self.create_dollar_fact()
-        self.check_transform()
+'''
+class DBTransformTest(BaseTest, MongoDBRequired):
+'''

awx/fact/utils/dbtransform.py

@@ -1,55 +1,55 @@
 # Copyright (c) 2014, Ansible, Inc.
 # All Rights Reserved.
+
+# Pymongo
 from pymongo.son_manipulator import SONManipulator
 
-'''
-Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802
-
-Replace . and $ with unicode values
-'''
 class KeyTransform(SONManipulator):
+
     def __init__(self, replace):
         self.replace = replace
 
-    def transform_key(self, key, replace, replacement):
-        """Transform key for saving to database."""
-        return key.replace(replace, replacement)
+    def replace_key(self, key):
+        for (replace, replacement) in self.replace:
+                key = key.replace(replace, replacement)
+        return key
 
-    def revert_key(self, key, replace, replacement):
-        """Restore transformed key returning from database."""
-        return key.replace(replacement, replace)
+    def revert_key(self, key):
+        for (replacement, replace) in self.replace:
+                key = key.replace(replace, replacement)
+        return key
+
+    def replace_incoming(self, obj):
+        if isinstance(obj, dict):
+            value = {}
+            for k, v in obj.items():
+                value[self.replace_key(k)] = self.replace_incoming(v)
+        elif isinstance(obj, list):
+            value = [self.replace_incoming(elem)
+                     for elem in obj]
+        else:
+            value = obj
+
+        return value
+
+    def replace_outgoing(self, obj):
+        if isinstance(obj, dict):
+            value = {}
+            for k, v in obj.items():
+                value[self.revert_key(k)] = self.replace_outgoing(v)
+        elif isinstance(obj, list):
+            value = [self.replace_outgoing(elem)
+                     for elem in obj]
+        else:
+            value = obj
+
+        return value
 
     def transform_incoming(self, son, collection):
-        """Recursively replace all keys that need transforming."""
-        for (key, value) in son.items():
-            for r in self.replace:
-                replace = r[0]
-                replacement = r[1]
-                if replace in key:
-                    if isinstance(value, dict):
-                        son[self.transform_key(key, replace, replacement)] = self.transform_incoming(
-                            son.pop(key), collection)
-                    else:
-                        son[self.transform_key(key, replace, replacement)] = son.pop(key)
-                elif isinstance(value, dict):  # recurse into sub-docs
-                    son[key] = self.transform_incoming(value, collection)
-        return son
+        return self.replace_incoming(son)
 
     def transform_outgoing(self, son, collection):
-        """Recursively restore all transformed keys."""
-        for (key, value) in son.items():
-            for r in self.replace:
-                replace = r[0]
-                replacement = r[1]
-                if replacement in key:
-                    if isinstance(value, dict):
-                        son[self.revert_key(key, replace, replacement)] = self.transform_outgoing(
-                            son.pop(key), collection)
-                    else:
-                        son[self.revert_key(key, replace, replacement)] = son.pop(key)
-                elif isinstance(value, dict):  # recurse into sub-docs
-                    son[key] = self.transform_outgoing(value, collection)
-        return son
+        return self.replace_outgoing(son)
 
 def register_key_transform(db):
     db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')]))