shaba/awx
1
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2024-10-30 22:21:13 +03:00
Code Releases Activity

Implemented fact scan storage logic.

Browse Source 
* added mongo connection logic
* added mongo dbtransform logic to allow keys with . and $
* altered tower fact scanner CacheModule to emit a message for each fact module facts (including ansible facts). Previously, seperate facts module facts were getting concatenated to each subsequent emi
* tower fact scanner CacheModule timeout set as to not hang for forever
* broke apart commands.py test
* added unit test for run_fact_cache_receiver, facts, and dbtransform
This commit is contained in:
Chris Meyers 2015-04-03 09:58:38 -04:00
parent a5452fa432
commit c03cef022d
19 changed files with 698 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
awx/main/dbtransform.py Normal file
View File

@ -0,0 +1,55 @@
# Copyright (c) 2014, Ansible, Inc.
# All Rights Reserved.
from pymongo.son_manipulator import SONManipulator
'''
Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802
Replace . and $ with unicode values
'''
class KeyTransform(SONManipulator):
def __init__(self, replace):
self.replace = replace
def transform_key(self, key, replace, replacement):
"""Transform key for saving to database."""
return key.replace(replace, replacement)
def revert_key(self, key, replace, replacement):
"""Restore transformed key returning from database."""
return key.replace(replacement, replace)
def transform_incoming(self, son, collection):
"""Recursively replace all keys that need transforming."""
for (key, value) in son.items():
for r in self.replace:
replace = r[0]
replacement = r[1]
if replace in key:
if isinstance(value, dict):
son[self.transform_key(key, replace, replacement)] = self.transform_incoming(
son.pop(key), collection)
else:
son[self.transform_key(key, replace, replacement)] = son.pop(key)
elif isinstance(value, dict): # recurse into sub-docs
son[key] = self.transform_incoming(value, collection)
return son
def transform_outgoing(self, son, collection):
"""Recursively restore all transformed keys."""
for (key, value) in son.items():
for r in self.replace:
replace = r[0]
replacement = r[1]
if replacement in key:
if isinstance(value, dict):
son[self.revert_key(key, replace, replacement)] = self.transform_outgoing(
son.pop(key), collection)
else:
son[self.revert_key(key, replace, replacement)] = son.pop(key)
elif isinstance(value, dict): # recurse into sub-docs
son[key] = self.transform_outgoing(value, collection)
return son
def register_key_transform(db):
db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')]))

121
awx/main/management/commands/run_fact_cache_receiver.py
View File

@ -2,88 +2,71 @@
# All Rights Reserved
import logging
from datetime import datetime
import json
from django.core.management.base import NoArgsCommand
from awx.main.models import * # noqa
from awx.main.socket import Socket
import pymongo
from pymongo import MongoClient
_MODULES = [ 'packages', 'services', 'files' ]
logger = logging.getLogger('awx.main.commands.run_fact_cache_receiver')
from pymongo.son_manipulator import SONManipulator
class KeyTransform(SONManipulator):
"""Transforms keys going to database and restores them coming out.
This allows keys with dots in them to be used (but does break searching on
them unless the find command also uses the transform.
Example & test:
# To allow `.` (dots) in keys
import pymongo
client = pymongo.MongoClient("mongodb://localhost")
db = client['delete_me']
db.add_son_manipulator(KeyTransform(".", "_dot_"))
db['mycol'].remove()
db['mycol'].update({'_id': 1}, {'127.0.0.1': 'localhost'}, upsert=True,
manipulate=True)
print db['mycol'].find().next()
print db['mycol'].find({'127_dot_0_dot_0_dot_1': 'localhost'}).next()
Note: transformation could be easily extended to be more complex.
"""
def __init__(self, replace, replacement):
self.replace = replace
self.replacement = replacement
def transform_key(self, key):
"""Transform key for saving to database."""
return key.replace(self.replace, self.replacement)
def revert_key(self, key):
"""Restore transformed key returning from database."""
return key.replace(self.replacement, self.replace)
def transform_incoming(self, son, collection):
"""Recursively replace all keys that need transforming."""
for (key, value) in son.items():
if self.replace in key:
if isinstance(value, dict):
son[self.transform_key(key)] = self.transform_incoming(
son.pop(key), collection)
else:
son[self.transform_key(key)] = son.pop(key)
elif isinstance(value, dict): # recurse into sub-docs
son[key] = self.transform_incoming(value, collection)
return son
def transform_outgoing(self, son, collection):
return son
class FactCacheReceiver(object):
def __init__(self):
self.client = MongoClient('localhost', 27017)
self.timestamp = None
def _determine_module(self, facts):
for x in _MODULES:
if x in facts:
return x
return 'ansible'
def _extract_module_facts(self, module, facts):
if module in facts:
f = facts[module]
return f
return facts
def process_facts(self, facts):
module = self._determine_module(facts)
facts = self._extract_module_facts(module, facts)
return (module, facts)
def process_fact_message(self, message):
host = message['host'].replace(".", "_")
facts = message['facts']
hostname = message['host']
facts_data = message['facts']
date_key = message['date_key']
host_db = self.client.host_facts
host_db.add_son_manipulator(KeyTransform(".", "_"))
host_db.add_son_manipulator(KeyTransform("$", "_"))
host_collection = host_db[host]
facts.update(dict(tower_host=host, datetime=date_key))
rec = host_collection.find({"datetime": date_key})
if rec.count():
this_fact = rec.next()
this_fact.update(facts)
host_collection.save(this_fact)
else:
host_collection.insert(facts)
# TODO: in ansible < v2 module_setup is emitted for "smart" fact caching.
# ansible v2 will not emit this message. Thus, this can be removed at that time.
if 'module_setup' in facts_data and len(facts_data) == 1:
return
try:
host = FactHost.objects.get(hostname=hostname)
except FactHost.DoesNotExist as e:
host = FactHost(hostname=hostname)
host.save()
except FactHost.MultipleObjectsReturned as e:
query = "db['fact_host'].find(hostname=%s)" % hostname
print('Database inconsistent. Multiple FactHost "%s" exist. Try the query %s to find the records.' % (hostname, query))
return
(module, facts) = self.process_facts(facts_data)
self.timestamp = datetime.fromtimestamp(date_key, None)
try:
# Update existing Fact entry
version_obj = FactVersion.objects.get(timestamp=self.timestamp, host=host, module=module)
Fact.objects(id=version_obj.fact.id).update_one(fact=facts)
except FactVersion.DoesNotExist:
# Create new Fact entry
(fact_obj, version_obj) = Fact.add_fact(self.timestamp, facts, host, module)
def run_receiver(self):
with Socket('fact_cache', 'r') as facts:

1
awx/main/models/__init__.py
View File

@ -16,6 +16,7 @@ from awx.main.models.ad_hoc_commands import * # noqa
from awx.main.models.schedules import * # noqa
from awx.main.models.activity_stream import * # noqa
from awx.main.models.ha import * # noqa
from awx.main.models.fact import * # noqa
# Monkeypatch Django serializer to ignore django-taggit fields (which break
# the dumpdata command; see https://github.com/alex/django-taggit/issues/155).

49
awx/main/models/fact.py Normal file
View File

@ -0,0 +1,49 @@
from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField
class FactHost(Document):
hostname = StringField(max_length=100, required=True, unique=True)
# TODO: Consider using hashed index on hostname. django-mongo may not support this but
# executing raw js will
meta = {
'indexes': [
'hostname'
]
}
class Fact(DynamicDocument):
timestamp = DateTimeField(required=True)
host = ReferenceField(FactHost, required=True)
module = StringField(max_length=50, required=True)
# fact = <anything>
# TODO: Consider using hashed index on host. django-mongo may not support this but
# executing raw js will
meta = {
'indexes': [
'-timestamp',
'host'
]
}
@staticmethod
def add_fact(timestamp, fact, host, module):
fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact)
fact_obj.save()
version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj)
version_obj.save()
return (fact_obj, version_obj)
class FactVersion(Document):
timestamp = DateTimeField(required=True)
host = ReferenceField(FactHost, required=True)
module = StringField(max_length=50, required=True)
fact = ReferenceField(Fact, required=True)
# TODO: Consider using hashed index on module. django-mongo may not support this but
# executing raw js will
meta = {
'indexes': [
'-timestamp',
'module'
]
}

3
awx/main/tests/__init__.py
View File

@ -15,3 +15,6 @@ from awx.main.tests.activity_stream import * # noqa
from awx.main.tests.schedules import * # noqa
from awx.main.tests.redact import * # noqa
from awx.main.tests.views import * # noqa
from awx.main.tests.models import * # noqa
from awx.main.tests.commands import * # noqa
from awx.main.tests.dbtransform import * # noqa

8
awx/main/tests/base.py
View File

@ -25,6 +25,9 @@ from django.contrib.auth.models import User
from django.test.client import Client
from django.test.utils import override_settings
# MongoEngine
from mongoengine.connection import get_db
# AWX
from awx.main.models import * # noqa
from awx.main.backend import LDAPSettings
@ -84,6 +87,11 @@ class BaseTestMixin(QueueTestMixin):
def setUp(self):
super(BaseTestMixin, self).setUp()
# Drop mongo database
self.db = get_db()
self.db.connection.drop_database(settings.MONGO_DB)
self.object_ctr = 0
# Save sys.path before tests.
self._sys_path = [x for x in sys.path]

5
awx/main/tests/commands/__init__.py Normal file
View File

@ -0,0 +1,5 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
from awx.main.tests.commands.run_fact_cache_receiver import * # noqa
from awx.main.tests.commands.commands_monolithic import * # noqa

89
awx/main/tests/commands/base.py Normal file
View File

@ -0,0 +1,89 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
import StringIO
import sys
import json
# Django
from django.core.management import call_command
# AWX
from awx.main.models import * # noqa
from awx.main.tests.base import BaseTestMixin
class BaseCommandMixin(BaseTestMixin):
'''
Base class for tests that run management commands.
'''
def create_test_inventories(self):
self.setup_users()
self.organizations = self.make_organizations(self.super_django_user, 2)
self.projects = self.make_projects(self.normal_django_user, 2)
self.organizations[0].projects.add(self.projects[1])
self.organizations[1].projects.add(self.projects[0])
self.inventories = []
self.hosts = []
self.groups = []
for n, organization in enumerate(self.organizations):
inventory = Inventory.objects.create(name='inventory-%d' % n,
description='description for inventory %d' % n,
organization=organization,
variables=json.dumps({'n': n}) if n else '')
self.inventories.append(inventory)
hosts = []
for x in xrange(10):
if n > 0:
variables = json.dumps({'ho': 'hum-%d' % x})
else:
variables = ''
host = inventory.hosts.create(name='host-%02d-%02d.example.com' % (n, x),
inventory=inventory,
variables=variables)
hosts.append(host)
self.hosts.extend(hosts)
groups = []
for x in xrange(5):
if n > 0:
variables = json.dumps({'gee': 'whiz-%d' % x})
else:
variables = ''
group = inventory.groups.create(name='group-%d' % x,
inventory=inventory,
variables=variables)
groups.append(group)
group.hosts.add(hosts[x])
group.hosts.add(hosts[x + 5])
if n > 0 and x == 4:
group.parents.add(groups[3])
self.groups.extend(groups)
def run_command(self, name, *args, **options):
'''
Run a management command and capture its stdout/stderr along with any
exceptions.
'''
command_runner = options.pop('command_runner', call_command)
stdin_fileobj = options.pop('stdin_fileobj', None)
options.setdefault('verbosity', 1)
options.setdefault('interactive', False)
original_stdin = sys.stdin
original_stdout = sys.stdout
original_stderr = sys.stderr
if stdin_fileobj:
sys.stdin = stdin_fileobj
sys.stdout = StringIO.StringIO()
sys.stderr = StringIO.StringIO()
result = None
try:
result = command_runner(name, *args, **options)
except Exception as e:
result = e
finally:
captured_stdout = sys.stdout.getvalue()
captured_stderr = sys.stderr.getvalue()
sys.stdin = original_stdin
sys.stdout = original_stdout
sys.stderr = original_stderr
return result, captured_stdout, captured_stderr

0
awx/main/tests/commands.py → awx/main/tests/commands/commands_monolithic.py
View File

199
awx/main/tests/commands/run_fact_cache_receiver.py Normal file
View File

@ -0,0 +1,199 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
# Python
import time
from datetime import datetime
import mock
import json
import unittest
from copy import deepcopy
from mock import Mock, MagicMock
# AWX
from awx.main.tests.base import BaseTest
from awx.main.tests.commands.base import BaseCommandMixin
from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver, _MODULES
from awx.main.models.fact import *
__all__ = ['RunFactCacheReceiverUnitTest', 'RunFactCacheReceiverFunctionalTest']
TEST_MSG_BASE = {
'host': 'hostname1',
'date_key': time.mktime(datetime.utcnow().timetuple()),
'facts' : { }
}
TEST_MSG_MODULES = {
'packages': {
"accountsservice": [
{
"architecture": "amd64",
"name": "accountsservice",
"source": "apt",
"version": "0.6.35-0ubuntu7.1"
}
],
"acpid": [
{
"architecture": "amd64",
"name": "acpid",
"source": "apt",
"version": "1:2.0.21-1ubuntu2"
}
],
"adduser": [
{
"architecture": "all",
"name": "adduser",
"source": "apt",
"version": "3.113+nmu3ubuntu3"
}
],
},
'services': [
{
"name": "acpid",
"source": "sysv",
"state": "running"
},
{
"name": "apparmor",
"source": "sysv",
"state": "stopped"
},
{
"name": "atd",
"source": "sysv",
"state": "running"
},
{
"name": "cron",
"source": "sysv",
"state": "running"
}
],
'ansible': {
'ansible_fact_simple': 'hello world',
'ansible_fact_complex': {
'foo': 'bar',
'hello': [
'scooby',
'dooby',
'doo'
]
},
}
}
# Derived from TEST_MSG_BASE
TEST_MSG = dict(TEST_MSG_BASE)
def copy_only_module(data, module):
data = deepcopy(data)
data['facts'] = {}
if module == 'ansible':
data['facts'] = deepcopy(TEST_MSG_MODULES[module])
else:
data['facts'][module] = deepcopy(TEST_MSG_MODULES[module])
return data
class RunFactCacheReceiverFunctionalTest(BaseCommandMixin, BaseTest):
@unittest.skip('''\
TODO: run_fact_cache_receiver enters a while True loop that never exists. \
This differs from most other commands that we test for. More logic and work \
would be required to invoke this case from the command line with little return \
in terms of increase coverage and confidence.''')
def test_invoke(self):
result, stdout, stderr = self.run_command('run_fact_cache_receiver')
self.assertEqual(result, None)
class RunFactCacheReceiverUnitTest(BaseTest):
# TODO: Check that timestamp and other attributes are as expected
def check_process_fact_message_module(self, data, module):
fact_found = None
facts = Fact.objects.all()
self.assertEqual(len(facts), 1)
for fact in facts:
if fact.module == module:
fact_found = fact
break
self.assertIsNotNone(fact_found)
#self.assertEqual(data['facts'][module], fact_found[module])
fact_found = None
fact_versions = FactVersion.objects.all()
self.assertEqual(len(fact_versions), 1)
for fact in fact_versions:
if fact.module == module:
fact_found = fact
break
self.assertIsNotNone(fact_found)
# Ensure that the message flows from the socket through to process_fact_message()
@mock.patch('awx.main.socket.Socket.listen')
def test_run_receiver(self, listen_mock):
listen_mock.return_value = [ TEST_MSG ]
receiver = FactCacheReceiver()
receiver.process_fact_message = MagicMock(name='process_fact_message')
receiver.run_receiver()
receiver.process_fact_message.assert_called_once_with(TEST_MSG)
def test_process_fact_message_ansible(self):
data = copy_only_module(TEST_MSG, 'ansible')
receiver = FactCacheReceiver()
receiver.process_fact_message(data)
self.check_process_fact_message_module(data, 'ansible')
def test_process_fact_message_packages(self):
data = copy_only_module(TEST_MSG, 'packages')
receiver = FactCacheReceiver()
receiver.process_fact_message(data)
self.check_process_fact_message_module(data, 'packages')
def test_process_fact_message_services(self):
data = copy_only_module(TEST_MSG, 'services')
receiver = FactCacheReceiver()
receiver.process_fact_message(data)
self.check_process_fact_message_module(data, 'services')
# Ensure that only a single host gets created for multiple invocations with the same hostname
def test_process_fact_message_single_host_created(self):
receiver = FactCacheReceiver()
data = deepcopy(TEST_MSG)
receiver.process_fact_message(data)
data = deepcopy(TEST_MSG)
data['date_key'] = time.mktime(datetime.utcnow().timetuple())
receiver.process_fact_message(data)
fact_hosts = FactHost.objects.all()
self.assertEqual(len(fact_hosts), 1)
def test_process_facts_message_ansible_overwrite(self):
data = copy_only_module(TEST_MSG, 'ansible')
key = 'ansible_overwrite'
value = 'hello world'
receiver = FactCacheReceiver()
receiver.process_fact_message(data)
fact = Fact.objects.all()[0]
data = copy_only_module(TEST_MSG, 'ansible')
data['facts'][key] = value
receiver.process_fact_message(data)
fact = Fact.objects.get(id=fact.id)
self.assertIn(key, fact.fact)
self.assertEqual(fact.fact[key], value)

78
awx/main/tests/dbtransform.py Normal file
View File

@ -0,0 +1,78 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
# Python
from datetime import datetime
from mongoengine.connection import get_db
from mongoengine import connect
# Django
from django.conf import settings
# AWX
from awx.main.tests.base import BaseTest
from awx.main.models.fact import *
__all__ = ['DBTransformTest']
class DBTransformTest(BaseTest):
def setUp(self):
super(DBTransformTest, self).setUp()
# Create a db connection that doesn't have the transformation registered
# Note: this goes through pymongo not mongoengine
self.client = connect(settings.MONGO_DB)
self.db = self.client[settings.MONGO_DB]
def _create_fact(self):
fact = {}
fact[self.k] = self.v
h = FactHost(hostname='blah')
h.save()
f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact)
f.save()
return f
def create_dot_fact(self):
self.k = 'this.is.a.key'
self.v = 'this.is.a.value'
self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey'
return self._create_fact()
def create_dollar_fact(self):
self.k = 'this$is$a$key'
self.v = 'this$is$a$value'
self.k_uni = 'this\uff04is\uff04a\uff04key'
return self._create_fact()
def check_unicode(self, f):
f_raw = self.db.fact.find_one(id=f.id)
self.assertIn(self.k_uni, f_raw['fact'])
self.assertEqual(f_raw['fact'][self.k_uni], self.v)
# Ensure key . are being transformed to the equivalent unicode into the database
def test_key_transform_dot_unicode_in_storage(self):
f = self.create_dot_fact()
self.check_unicode(f)
# Ensure key $ are being transformed to the equivalent unicode into the database
def test_key_transform_dollar_unicode_in_storage(self):
f = self.create_dollar_fact()
self.check_unicode(f)
def check_transform(self):
f = Fact.objects.all()[0]
self.assertIn(self.k, f.fact)
self.assertEqual(f.fact[self.k], self.v)
def test_key_transform_dot_on_retreive(self):
self.create_dot_fact()
self.check_transform()
def test_key_transform_dollar_on_retreive(self):
self.create_dollar_fact()
self.check_transform()

3
awx/main/tests/jobs/__init__.py
View File

@ -1,3 +1,6 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
from awx.main.tests.jobs.jobs_monolithic import * # noqa
from survey_password import * # noqa
from base import * # noqa

4
awx/main/tests/models/__init__.py Normal file
View File

@ -0,0 +1,4 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
from awx.main.tests.models.fact import * # noqa

87
awx/main/tests/models/fact.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
# Python
from datetime import datetime
# Django
# AWX
from awx.main.models.fact import *
from awx.main.tests.base import BaseTest
__all__ = ['FactHostTest', 'FactTest']
TEST_FACT_DATA = {
'hostname': 'hostname1',
'add_fact_data': {
'timestamp': datetime.now(),
'host': None,
'module': 'packages',
'fact': {
"accountsservice": [
{
"architecture": "amd64",
"name": "accountsservice",
"source": "apt",
"version": "0.6.35-0ubuntu7.1"
}
],
"acpid": [
{
"architecture": "amd64",
"name": "acpid",
"source": "apt",
"version": "1:2.0.21-1ubuntu2"
}
],
"adduser": [
{
"architecture": "all",
"name": "adduser",
"source": "apt",
"version": "3.113+nmu3ubuntu3"
}
],
},
}
}
# Strip off microseconds because mongo has less precision
TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0)
class FactHostTest(BaseTest):
def test_create_host(self):
host = FactHost(hostname=TEST_FACT_DATA['hostname'])
host.save()
host = FactHost.objects.get(hostname=TEST_FACT_DATA['hostname'])
self.assertIsNotNone(host, "Host added but not found")
self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname")
class FactTest(BaseTest):
def setUp(self):
super(FactTest, self).setUp()
TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save()
def test_add_fact(self):
(f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data'])
f = Fact.objects.get(id=f_obj.id)
v = FactVersion.objects.get(id=v_obj.id)
self.assertEqual(f.id, f_obj.id)
self.assertEqual(f.module, TEST_FACT_DATA['add_fact_data']['module'])
self.assertEqual(f.fact, TEST_FACT_DATA['add_fact_data']['fact'])
self.assertEqual(f.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp'])
# host relationship created
self.assertEqual(f.host.id, TEST_FACT_DATA['add_fact_data']['host'].id)
# version created and related
self.assertEqual(v.id, v_obj.id)
self.assertEqual(v.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp'])
self.assertEqual(v.host.id, TEST_FACT_DATA['add_fact_data']['host'].id)
self.assertEqual(v.fact.id, f_obj.id)
self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module'])

52
awx/plugins/fact_caching/tower.py
View File

@ -32,6 +32,8 @@
import sys
import time
import datetime
import json
from copy import deepcopy
from ansible import constants as C
from ansible.cache.base import BaseCacheModule
@ -47,6 +49,7 @@ class CacheModule(BaseCacheModule):
# Basic in-memory caching for typical runs
self._cache = {}
self._cache_prev = {}
# This is the local tower zmq connection
self._tower_connection = C.CACHE_PLUGIN_CONNECTION
@ -54,20 +57,67 @@ class CacheModule(BaseCacheModule):
try:
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.RCVTIMEO, 4000)
self.socket.setsockopt(zmq.LINGER, 2000)
self.socket.connect(self._tower_connection)
except Exception, e:
print("Connection to zeromq failed at %s with error: %s" % (str(self._tower_connection),
str(e)))
sys.exit(1)
def identify_ansible_facts(self, facts):
ansible_keys = {}
for k in facts.keys():
if k.startswith('ansible_'):
ansible_keys[k] = 1
return ansible_keys
def identify_new_module(self, key, value):
if key in self._cache_prev:
value_old = self._cache_prev[key]
for k,v in value.iteritems():
if k not in value_old:
if not k.startswith('ansible_'):
return k
return None
def get(self, key):
return self._cache.get(key)
'''
get() returns a reference to the fact object (usually a dict). The object is modified directly,
then set is called. Effectively, pre-determining the set logic.
The below logic creates a backup of the cache each set. The values are now preserved across set() calls.
For a given key. The previous value is looked at for new keys that aren't of the form 'ansible_'.
If found, send the value of the found key.
If not found, send all the key value pairs of the form 'ansible_' (we presume set() is called because
of an ansible fact module invocation)
More simply stated...
In value, if a new key is found at the top most dict then consider this a module request and only
emit the facts for the found top-level key.
If a new key is not found, assume set() was called as a result of ansible facts scan. Thus, emit
all facts of the form 'ansible_'.
'''
def set(self, key, value):
module = self.identify_new_module(key, value)
# Assume ansible fact triggered the set if no new module found
facts = {}
if not module:
keys = self.identify_ansible_facts(value)
for k in keys:
facts[k] = value[k]
else:
facts[module] = value[module]
self._cache_prev = deepcopy(self._cache)
self._cache[key] = value
# Emit fact data to tower for processing
self.socket.send_json(dict(host=key, facts=value, date_key=self.date_key))
self.socket.send_json(dict(host=key, facts=facts, date_key=self.date_key))
self.socket.recv()
def keys(self):

8
awx/settings/__init__.py
View File

@ -1,2 +1,10 @@
# Copyright (c) 2014 AnsibleWorks, Inc.
# All Rights Reserved.
from django.conf import settings
from mongoengine import connect
from mongoengine.connection import get_db
from awx.main.dbtransform import register_key_transform
connect(settings.MONGO_DB)
register_key_transform(get_db())

2
awx/settings/defaults.py
View File

@ -7,6 +7,8 @@ import glob
from datetime import timedelta
import tempfile
MONGO_DB = 'system_tracking'
# Update this module's local settings from the global settings module.
from django.conf import global_settings
this_module = sys.modules[__name__]

2
awx/settings/development.py
View File

@ -14,6 +14,8 @@ from split_settings.tools import optional, include
# Load default settings.
from defaults import *
MONGO_DB = 'system_tracking_dev'
# Disable capturing all SQL queries when running celeryd in development.
if 'celeryd' in sys.argv:
SQL_DEBUG = False

3
requirements/dev_local.txt
View File

@ -55,7 +55,6 @@ mongo-python-driver-2.8.tar.gz
# Needed by pyrax:
#httplib2-0.8.tar.gz
#keyring-3.7.zip
#mock-1.0.1.tar.gz
#python-swiftclient-2.0.3.tar.gz
#rackspace-novaclient-1.4.tar.gz
# Remaining dev/prod packages:
@ -78,6 +77,8 @@ mongo-python-driver-2.8.tar.gz
#mongoengine-0.9.0.tar.gz
# Dev-only packages:
# Needed for tests
mock-1.0.1.tar.gz
# Needed by django-debug-toolbar:
sqlparse-0.1.11.tar.gz
# Needed for Python2.6 support: