F #3337: Retry if failure during shutdown (#3338)

* F #3337: Retry if failure during shutdown

* F #3337: Configurable number of retries

* F #3337: Extend to deploy and cancel

* F #3337: Use retry_interval variable

* Update shutdown

src/vmm_mad/remotes/vcenter/cancel

@@ -29,6 +29,8 @@ $LOAD_PATH << File.dirname(__FILE__)
 
 require 'vcenter_driver'
 
+CONFIG = VCenterConf.new
+
 vm_ref    = ARGV[0]
 host      = ARGV[1]
 vm_id     = ARGV[-2]
@@ -43,16 +45,26 @@ check_valid(lcm_state, 'lcm_state')
 lcm_state_str = OpenNebula::VirtualMachine::LCM_STATE[lcm_state.to_i]
 
 begin
+    retries ||= 0
     vi_client = VCenterDriver::VIClient.new_from_host(host_id)
 
     vm = VCenterDriver::VirtualMachine.new(vi_client, vm_ref, vm_id)
 
     vm.poweroff_hard
 rescue StandardError => e
-    message = "Cancel VM #{vm_ref} failed due to "\
-              "\"#{e.message}\"\n#{e.backtrace.join("\n")}"
+    if (retries += 1) < CONFIG[:retries]
+        message = "Cancel VM #{vm_ref} failed due to "\
+                  "\"#{e.message}\"\n#{e.backtrace.join("\n")} "\
+                  "on the attempt \##{retries}\n#{e.backtrace.join("\n")}"
+    else
+        message = "Cancel VM #{vm_ref} failed due to "\
+                  "\"#{e.message}\"\n#{e.backtrace.join("\n")} "\
+                  "on the final attempt\n#{e.backtrace.join("\n")}"
+    end
     OpenNebula.log_error(message)
     STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+    sleep CONFIG[:retry_interval].to_i
+    retry if retries < CONFIG[:retries]
 
     exit(-1)
 ensure

src/vmm_mad/remotes/vcenter/deploy

@@ -29,6 +29,8 @@ $LOAD_PATH << File.dirname(__FILE__)
 
 require 'vcenter_driver'
 
+CONFIG = VCenterConf.new
+
 dfile        = ARGV[0]
 cluster_name = ARGV[1]
 vm_id        = ARGV[2]
@@ -47,6 +49,7 @@ host_id      = drv_action['HISTORY_RECORDS/HISTORY/HID']
 deploy       = {}
 
 begin
+    retries ||= 0
     vi_client = VCenterDriver::VIClient.new_from_host(host_id)
     one_vm = VCenterDriver::VIHelper.one_item(OpenNebula::VirtualMachine, vm_id)
 
@@ -67,10 +70,19 @@ begin
 
     puts vm['_ref']
 rescue StandardError => e
-    message =  "Deploy of VM #{vm_id} on vCenter cluster #{cluster_name} " \
-               "with #{dfile} failed due to \"#{e.message}\"."
+    if (retries += 1) < CONFIG[:retries]
+        message =  "Deploy of VM #{vm_id} on vCenter cluster #{cluster_name} " \
+                   "with #{dfile} failed due to \"#{e.message}\"" \
+                   "on the attempt \##{retries}."
+    else
+        message =  "Deploy of VM #{vm_id} on vCenter cluster #{cluster_name} " \
+                   "with #{dfile} failed due to \"#{e.message}\"" \
+                   "on the final attempt."
+    end
     OpenNebula.log_error(message)
     STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+    sleep CONFIG[:retry_interval].to_i
+    retry if retries < CONFIG[:retries]
 
     exit(-1)
 ensure

src/vmm_mad/remotes/vcenter/shutdown

@@ -29,6 +29,8 @@ $LOAD_PATH << File.dirname(__FILE__)
 
 require 'vcenter_driver'
 
+CONFIG = VCenterConf.new
+
 vm_ref          = ARGV[0]
 vc_cluster_name = ARGV[1]
 vm_id           = ARGV[2]
@@ -50,6 +52,7 @@ if !(%{'SAVE_MIGRATE', 'SHUTDOWN', 'SHUTDOWN_POWEROFF', 'SHUTDOWN_UNDEPLOY'}).in
 end
 
 begin
+    retries ||= 0
     vi_client = VCenterDriver::VIClient.new_from_host(host_id)
 
     if vm_ref.empty?
@@ -63,11 +66,19 @@ begin
 
     vm.shutdown # Undeploy, Poweroff or Terminate
 rescue StandardError => e
-    message = "Shutdown of VM #{vm_ref} on vCenter cluster "\
-              "#{vc_cluster_name} failed due to "\
-              "\"#{e.message}\"\n#{e.backtrace.join("\n")}"
+    if (retries += 1) < CONFIG[:retries]
+        message = "Shutdown of VM #{vm_ref} on vCenter cluster "\
+                  "#{vc_cluster_name} failed due to "\
+                  "\"#{e.message}\" on the attempt \##{retries}\n#{e.backtrace.join("\n")}"
+    else
+        message = "Shutdown of VM #{vm_ref} on vCenter cluster "\
+                  "#{vc_cluster_name} failed due to "\
+                  "\"#{e.message}\" on the final attempt\n#{e.backtrace.join("\n")}"
+    end
     OpenNebula.log_error(message)
     STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+    sleep CONFIG[:retry_interval].to_i
+    retry if retries < CONFIG[:retries]
 
     exit(-1)
 ensure

src/vmm_mad/remotes/vcenter/vcenter_driver.rb

@@ -44,7 +44,9 @@ class VCenterConf < Hash
     DEFAULT_CONFIGURATION = {
         :delete_images => false,
         :vm_poweron_wait_default => 300,
-        :debug_information => false
+        :debug_information => false,
+        :retries => 3,
+        :retry_interval => 1
     }
 
     def initialize