Merge pull request #4183 from ryanpetrello/logging-deadlock

don't ship external logs from the main thread of the dispatcher

Reviewed-by: https://github.com/softwarefactory-project-zuul[bot]

awx/main/management/commands/run_dispatcher.py

@@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand
 from django.db import connection as django_connection, connections
 from kombu import Exchange, Queue
 
+from awx.main.utils.handlers import AWXProxyHandler
 from awx.main.dispatch import get_local_queuename, reaper
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.kombu import Connection
@@ -121,6 +122,12 @@ class Command(BaseCommand):
 
         reaper.reap()
         consumer = None
+
+        # don't ship external logs inside the dispatcher's parent process
+        # this exists to work around a race condition + deadlock bug on fork
+        # in cpython itself:
+        # https://bugs.python.org/issue37429
+        AWXProxyHandler.disable()
         with Connection(settings.BROKER_URL) as conn:
             try:
                 bcast = 'tower_broadcast_all'

awx/main/utils/handlers.py

@@ -6,6 +6,7 @@ import logging
 import json
 import requests
 import time
+import threading
 import socket
 import select
 from urllib import parse as urlparse
@@ -286,6 +287,8 @@ class AWXProxyHandler(logging.Handler):
     Parameters match same parameters in the actualized handler classes.
     '''
 
+    thread_local = threading.local()
+
     def __init__(self, **kwargs):
         # TODO: process 'level' kwarg
         super(AWXProxyHandler, self).__init__(**kwargs)
@@ -322,8 +325,9 @@ class AWXProxyHandler(logging.Handler):
         return self._handler
 
     def emit(self, record):
-        actual_handler = self.get_handler()
-        return actual_handler.emit(record)
+        if AWXProxyHandler.thread_local.enabled:
+            actual_handler = self.get_handler()
+            return actual_handler.emit(record)
 
     def perform_test(self, custom_settings):
         """
@@ -353,6 +357,13 @@ class AWXProxyHandler(logging.Handler):
             except RequestException as e:
                 raise LoggingConnectivityException(str(e))
 
+    @classmethod
+    def disable(cls):
+        cls.thread_local.enabled = False
+
+
+AWXProxyHandler.thread_local.enabled = True
+
 
 ColorHandler = logging.StreamHandler