shaba/awx
1
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2024-10-31 06:51:10 +03:00
Code Releases Activity

sleep backoff on cb receiver reconnect

Browse Source 
* Sleep before trying to reconnect
Most common reason for entering this reconnect loop is when Redis
service stops before the callback receiver when stopping tower services.
This commit is contained in:
chris meyers 2020-04-28 12:24:34 -04:00
parent 0d30a67756
commit 2ecd055d1e
1 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
awx/main/dispatch/worker/base.py
View File

@ -8,6 +8,7 @@ import sys
import redis
import json
import psycopg2
import time
from uuid import UUID
from queue import Empty as QueueEmpty
@ -116,18 +117,23 @@ class AWXConsumerRedis(AWXConsumerBase):
super(AWXConsumerRedis, self).run(*args, **kwargs)
self.worker.on_start()
queue = redis.Redis.from_url(settings.BROKER_URL)
retry = 0
while True:
try:
res = queue.blpop(self.queues)
res = json.loads(res[1])
self.process_task(res)
except redis.exceptions.RedisError:
logger.exception("encountered an error communicating with redis")
except (json.JSONDecodeError, KeyError):
logger.exception("failed to decode JSON message from redis")
if self.should_stop:
return
queue = redis.Redis.from_url(settings.BROKER_URL)
while True:
try:
res = queue.blpop(self.queues)
retry = 0
res = json.loads(res[1])
self.process_task(res)
except redis.exceptions.RedisError:
logger.exception(f"encountered an error communicating with redis. Reconnect attempt {retry}")
retry += 1
time.sleep(min(retry * 2, 30))
except (json.JSONDecodeError, KeyError):
logger.exception("failed to decode JSON message from redis")
if self.should_stop:
return
class AWXConsumerPG(AWXConsumerBase):