Merge pull request #5721 from wenottingham/a-thundering-herd-is-only-good-if-you-are-marshall

Change how analytics is gathered to only gather once per interval

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/conf/fields.py

@@ -11,7 +11,7 @@ from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework.fields import (  # noqa
-    BooleanField, CharField, ChoiceField, DictField, EmailField,
+    BooleanField, CharField, ChoiceField, DictField, DateTimeField, EmailField,
     IntegerField, ListField, NullBooleanField
 )
 

awx/main/conf.py

@@ -799,6 +799,27 @@ register(
 )
 
 
+register(
+    'AUTOMATION_ANALYTICS_LAST_GATHER',
+    field_class=fields.DateTimeField,
+    label=_('Last gather date for Automation Analytics'),
+    allow_null=True,
+    category=_('System'),
+    category_slug='system'
+)
+
+
+register(
+    'AUTOMATION_ANALYTICS_GATHER_INTERVAL',
+    field_class=fields.IntegerField,
+    label=_('Interval (in seconds) between data gathering'),
+    default=14400,	# every 4 hours
+    min_value=1800,	# every 30 minutes
+    category=_('System'),
+    category_slug='system'
+)
+
+
 def logging_validate(serializer, attrs):
     if not serializer.instance or \
             not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or \

awx/main/tasks.py

@@ -339,15 +339,23 @@ def send_notifications(notification_list, job_id=None):
 def gather_analytics():
     if not settings.INSIGHTS_TRACKING_STATE:
         return
-    try:
+    last_time = settings.AUTOMATION_ANALYTICS_LAST_GATHER
-        tgz = analytics.gather()
+    gather_time = now()
-        if not tgz:
+    if not last_time or ((gather_time - last_time).seconds > settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
-            return
+        with advisory_lock('gather_analytics_lock', wait=False) as acquired:
-        logger.debug('gathered analytics: {}'.format(tgz))
+            if acquired is False:
-        analytics.ship(tgz)
+                logger.debug('Not gathering analytics, another task holds lock')
-    finally:
+                return
-        if os.path.exists(tgz):
+            try:
-            os.remove(tgz)
+                tgz = analytics.gather()
+                if not tgz:
+                    return
+                logger.info('gathered analytics: {}'.format(tgz))
+                analytics.ship(tgz)
+                settings.AUTOMATION_ANALYTICS_LAST_GATHER = gather_time
+            finally:
+                if os.path.exists(tgz):
+                    os.remove(tgz)
 
 
 @task(queue=get_local_queuename)

awx/settings/defaults.py

@@ -5,8 +5,6 @@ import os
 import re  # noqa
 import sys
 from datetime import timedelta
-from celery.schedules import crontab
-import random
 
 # global settings
 from django.conf import global_settings
@@ -442,7 +440,7 @@ CELERYBEAT_SCHEDULE = {
     },
     'gather_analytics': {
         'task': 'awx.main.tasks.gather_analytics',
-        'schedule': crontab(hour='*/6', minute=random.randint(0,59))
+        'schedule': timedelta(minutes=5)
     },
     'task_manager': {
         'task': 'awx.main.scheduler.tasks.run_task_manager',
@@ -666,6 +664,9 @@ PENDO_TRACKING_STATE = "off"
 # Note: This setting may be overridden by database settings.
 INSIGHTS_TRACKING_STATE = False
 
+# Last gather date for Analytics
+AUTOMATION_ANALYTICS_LAST_GATHER = None
+AUTOMATION_ANALYTICS_INTERVAL = 14400
 
 # Default list of modules allowed for ad hoc commands.
 # Note: This setting may be overridden by database settings.

awx/ui/client/src/configuration/forms/system-form/sub-forms/system-misc.form.js

@@ -79,6 +79,12 @@ export default ['i18n', function(i18n) {
             AUTOMATION_ANALYTICS_URL: {
                 type: 'text',
                 reset: 'AUTOMATION_ANALYTICS_URL',
+            },
+            AUTOMATION_ANALYTICS_GATHER_INTERVAL: {
+                type: 'number',
+                integer: true,
+                min: 1800,
+                reset: 'AUTOMATION_ANALYTICS_GATHER_INTERVAL',
             }
         },