F #4302: Making vcenter_monitor more robust

src/im_mad/im_exec/one_im_sh

@@ -24,4 +24,9 @@ fi
 
 export DRIVER_NAME="one_im_sh_${BASH_ARGV##* }"
 
+# Explicitly execute the vCenter monitor driver in the background
+# This way we don't have to wait until OpenNebula
+# issues the add_hosts commands to know about missing hosts
+#(/usr/bin/env ruby $VAR_LOCATION/remotes/im/lib/vcenter_monitor.rb &)
+
 exec $MAD_LOCATION/one_im_exec -l $*

src/im_mad/remotes/lib/vcenter_cluster.rb

@@ -100,6 +100,7 @@ class Cluster
     def initialize(hid, onec)
         @host = OpenNebula::Host.new_with_id(hid, onec)
         @onec = onec
+        @hid = hid
 
         rc = @host.info(true)
 
@@ -109,10 +110,16 @@ class Cluster
 
         @monitordc = nil
 
-        #-----------------------------------------------------------------------
-        #  VI Client Initialization
-        #-----------------------------------------------------------------------
-        @vic     = VCenterDriver::VIClient.new(connection, hid)
+        connect_vcenter
+    end
+
+    #-----------------------------------------------------------------------
+    #  VI Client Initialization
+    #-----------------------------------------------------------------------
+    def connect_vcenter
+        # Avoid leaving open sessions to vCenter
+        @vic.close_connection if @vic
+        @vic     = VCenterDriver::VIClient.new(connection, @hid)
         @cluster = VCenterDriver::ClusterComputeResource
                    .new_from_ref(connection[:ccr], @vic)
     end
@@ -814,6 +821,11 @@ class ClusterSet
                     probe_frequency = conf[probe_name].to_i
                     next unless (Time.now.to_i - last_mon) > probe_frequency
 
+                    # Refresh the vCenter connection in the least frequent probe
+                    if probe_name.eql?('system_host')
+                        c[:cluster].connect_vcenter
+                    end
+
                     $logger.info("\tRunning #{probe_name} probe")
 
                     begin

src/im_mad/remotes/lib/vcenter_monitor.rb

@@ -38,7 +38,6 @@ $LOAD_PATH << RUBY_LIB_LOCATION
 require 'yaml'
 require 'rexml/document'
 
-
 require_relative './vcenter_cluster'
 require_relative './monitord_client'
 
@@ -47,10 +46,12 @@ require_relative './monitord_client'
 #
 #---------------------------------------------------------------------------
 class VcenterMonitorManager
+
     #---------------------------------------------------------------------------
     # Constants
     #    BASE_TIMER: for monitor loop, periods CANNOT be less than this value
-    #    DEFAULT_CONFIGURATION
+    #    DEFAULT_CONFIGURATION: hash with probes intervals and endpoint
+    #    MINIMUM_INTERVAL: minimum value for the interval of any of the probes
     #---------------------------------------------------------------------------
     BASE_TIMER = 10
 
@@ -64,16 +65,15 @@ class VcenterMonitorManager
         :port         => 4124
     }.freeze
 
-    # Minimum interval for vCenter probes
-    # They run in the front-end and are RAM consuming
-    # so this is the minimum grain for the intervals
     MINIMUM_INTERVAL = 30
 
-
     #---------------------------------------------------------------------------
     #
     #---------------------------------------------------------------------------
     def initialize
+        # Sanitize previous instances
+        kill_other_instances
+
         @clusters = ClusterSet.new
 
         @clusters.bootstrap
@@ -85,6 +85,17 @@ class VcenterMonitorManager
         Thread.new { timer }
     end
 
+    # Kills previous monitor instances
+    def kill_other_instances
+        path = File.expand_path(__FILE__)
+        ps_str = "ps auxwww|grep \"ruby #{path}\"|grep -v grep|awk {'print $2'}"
+        ids = `#{ps_str}`
+        return if ids.nil?
+
+        ids = ids.split("\n").map{|pid| pid.to_i}
+        ids.each{ |pid| Process.kill('KILL', pid) if pid!=Process.pid }
+    end
+
     #---------------------------------------------------------------------------
     #
     #---------------------------------------------------------------------------
@@ -102,6 +113,7 @@ class VcenterMonitorManager
                 :port         => conf.elements['NETWORK/PORT'].text.to_s
             }
 
+            # Don't allow intervals lower than the minimum default
             @conf.each{ |k,v| @conf[k] = 30 if v.is_a?(Integer) && v < MINIMUM_INTERVAL }
         end