F #3380: Retry hem execution. Better hook management start process

(cherry picked from commit cb6ecaa8bebf4c8e2323ac35a5ff601f29b97c37)
(cherry picked from commit b8351898fc212065dc1abff14be531db5442cd82)

share/pkgs/CentOS7/opennebula.service

@@ -2,7 +2,7 @@
 Description=OpenNebula Cloud Controller Daemon
 After=syslog.target network.target remote-fs.target
 After=mariadb.service mysql.service
-Wants=opennebula-scheduler.service
+Wants=opennebula-scheduler.service opennebula-hem.service
 
 [Service]
 Type=notify

share/scripts/one

@@ -19,27 +19,33 @@
 if [ -z "$ONE_LOCATION" ]; then
     ONE_PID=/var/run/one/oned.pid
     ONE_SCHEDPID=/var/run/one/sched.pid
+    ONE_HEMPID=/var/run/one/hem.pid
     ONE_CONF=/etc/one/oned.conf
     ONE_DB=/var/lib/one/one.db
     ONE_LOG=/var/log/one/oned.log
     ONE_SCHED_LOG=/var/log/one/sched.log
+    ONE_HEM_LOG=/var/log/one/onehem.log
     ONE_XMLRPC_LOG=/var/log/one/one_xmlrpc.log
 
     ONED=/usr/bin/oned
     ONE_SCHEDULER=/usr/bin/mm_sched
+    ONE_HEM=/usr/bin/onehem-server
 
     LOCK_FILE=/var/lock/one/one
 else
     ONE_PID=$ONE_LOCATION/var/oned.pid
     ONE_SCHEDPID=$ONE_LOCATION/var/sched.pid
+    ONE_HEMPID=$ONE_LOCATION/var/hem.pid
     ONE_CONF=$ONE_LOCATION/etc/oned.conf
     ONE_DB=$ONE_LOCATION/var/one.db
     ONE_LOG=$ONE_LOCATION/var/oned.log
     ONE_SCHED_LOG=$ONE_LOCATION/var/sched.log
+    ONE_HEM_LOG=$ONE_LOCATION/var/onehem.log
     ONE_XMLRPC_LOG=$ONE_LOCATION/var/one_xmlrpc.log
 
     ONED=$ONE_LOCATION/bin/oned
     ONE_SCHEDULER=$ONE_LOCATION/bin/mm_sched
+    ONE_HEM=$ONE_LOCATION/bin/onehem-server
 
     LOCK_FILE=$ONE_LOCATION/var/.lock
 fi
@@ -74,10 +80,11 @@ setup()
             ONESCHEDPID=`cat $ONE_SCHEDPID`
             ps $ONESCHEDPID > /dev/null 2>&1
             if [ $? -eq 0 ]; then
-                echo "The scheduler is still running (PID:$ONEPID). Please try 'one stop' first."
+                echo "The scheduler is still running (PID:$ONESCHEDPID). Please try 'one stop' first."
                 exit 1
             fi
         fi
+
         echo "Stale .lock detected. Erasing it."
         rm $LOCK_FILE
     fi
@@ -91,6 +98,8 @@ stop()
     stop_oned
 
     stop_sched
+
+    stop_hem
 }
 
 stop_oned()
@@ -121,6 +130,11 @@ stop_sched()
     fi
 }
 
+stop_hem()
+{
+    onehem-server stop
+}
+
 #------------------------------------------------------------------------------
 # Function that starts the daemons
 #------------------------------------------------------------------------------
@@ -136,6 +150,11 @@ start()
         exit 1
     fi
 
+    if [ ! -x "$ONE_HEM" ]; then
+        echo "Can not find $ONE_HEM."
+        exit 1
+    fi
+
     if [ ! -f "$ONE_DB" ]; then
         if [ ! -f "$HOME/.one/one_auth" ]; then
             if [ -z "$ONE_AUTH" ]; then
@@ -153,6 +172,9 @@ start()
     # Start the scheduler
     start_sched
 
+    # Start hook execution manager server
+    start_hem
+
     # Wait for the daemons to warm up
     sleep 3
 
@@ -215,6 +237,20 @@ start_sched()
     fi
 }
 
+start_hem()
+{
+    if [ "$BACKUP" = "true" ];then
+        [ -f "$ONE_HEM_LOG" ] && mv $ONE_HEM_LOG{,.$(date '+%Y%m%d%H%M%S')}
+    fi
+
+    LASTRC=$?
+
+    if [ $LASTRC -ne 0 ]; then
+        echo "Error starting onehem-server"
+        exit 1
+    fi
+}
+
 #------------------------------------------------------------------------------
 #------------------------------------------------------------------------------
 
@@ -230,13 +266,13 @@ case "$1" in
         ;;
     stop)
         stop
-        echo "oned and scheduler stopped"
+        echo "oned, scheduler and hem stopped"
         ;;
     restart)
         stop
         setup
         start
-        echo "oned and scheduler restarted"
+        echo "oned, scheduler and hem restarted"
         ;;
     start-sched)
         start_sched
@@ -250,7 +286,7 @@ case "$1" in
         start_sched
         ;;
     *)
-        echo "Usage: one [-f] {start|stop|restart|start-sched|stop-sched|restart-sched}" >&2
+        echo "Usage: one [-f] {start|stop|restart|start-sched|stop-sched|restart-sched|start-hem|stop-hem|restart-hem}" >&2
         echo "Options:" >&2
         echo "  -f  Do not backup log files." >&2
         exit 3

src/hem/onehem-server.rb

@@ -53,6 +53,9 @@ require 'opennebula'
 require 'CommandManager'
 require 'ActionManager'
 
+# Number of retries for loading hook information
+RETRIES = 300
+
 #-------------------------------------------------------------------------------
 #-------------------------------------------------------------------------------
 # This module includes basic functions to deal with Hooks
@@ -231,15 +234,6 @@ class HookMap
     def load
         @logger.info('Loading Hooks...')
 
-        hook_pool = OpenNebula::HookPool.new(@client)
-
-        rc = hook_pool.info
-
-        if OpenNebula.is_error?(rc)
-            @logger.error("Cannot get hook information: #{rc.message}")
-            return
-        end
-
         @hooks   = {}
         @filters = {}
 
@@ -247,6 +241,21 @@ class HookMap
             @hooks[type] = {}
         end
 
+        hook_pool = OpenNebula::HookPool.new(@client)
+        rc = nil
+
+        RETRIES.times do
+            rc = hook_pool.info
+            break unless OpenNebula.is_error?(rc)
+
+            sleep 0.5
+        end
+
+        if OpenNebula.is_error?(rc)
+            @logger.error("Cannot get hook information: #{rc.message}")
+            return
+        end
+
         hook_pool.each do |hook|
             hook.extend(HEMHook)
 
@@ -258,7 +267,8 @@ class HookMap
             key = hook.key
 
             @hooks[hook.type][key] = hook
-            @hooks_id[hook.id] = hook
+            @hooks_id[hook.id]     = hook
+
             @filters[hook['ID'].to_i] = hook.filter(key)
         end
 
@@ -448,6 +458,8 @@ class HookExecutionManager
             @subscriber.recv_string(key)
             @subscriber.recv_string(content)
 
+            @logger.debug("New event receive\nkey: #{key}\ncontent: #{content}")
+
             # get action
             action = key.split(' ').shift.to_sym