Merge branch 'bug-1211'

src/mad/sh/scripts_common.sh

@@ -176,6 +176,48 @@ function timeout_exec_and_log
     fi
 }
 
+# Parameters are times (seconds) and monitoring command (or function).
+# Executes monitoring command until it is successful (VM is no longer
+# running) or the timeout is reached.
+function retry
+{
+    times=$1
+    function=$2
+
+    count=1
+
+    ret=$($function)
+    error=$?
+
+    while [ $count -lt $times -a "$error" != "0" ]; do
+        sleep 1
+        count=$(( $count + 1 ))
+        ret=$($function)
+        error=$?
+    done
+
+    [ "x$error" = "x0" ]
+}
+
+# Parameters are deploy_id and cancel command. If the last command is
+# unsuccessful and $FORCE_DESTROY=yes then calls cancel command
+function force_shutdown {
+    error=$?
+    deploy_id=$1
+    command=$2
+
+    if [ "x$error" != "x0" ]; then
+        if [ "$FORCE_DESTROY" = "yes" ]; then
+            log_error "Timeout shutting down $deploy_id. Destroying it"
+            $($command)
+            sleep 2
+        else
+            error_message "Timed out shutting down $deploy_id"
+            exit -1
+        fi
+    fi
+}
+
 # This function will return a command that upon execution will format a
 # filesystem with its proper parameters based on the filesystem type
 function mkfs_command {

src/vmm_mad/remotes/kvm/kvmrc

@@ -19,3 +19,10 @@ export LANG=C
 export LIBVIRT_URI=qemu:///system
 
 export QEMU_PROTOCOL=qemu+ssh
+
+# Senconds to wait after shutdown until timeout
+export SHUTDOWN_TIMEOUT=120
+
+# Uncoment this line to force VM cancellation after shutdown timeout
+#export FORCE_DESTROY=yes
+

src/vmm_mad/remotes/kvm/shutdown

@@ -1,4 +1,4 @@
-#!/bin/bash 
+#!/bin/bash
 
 # -------------------------------------------------------------------------- #
 # Copyright 2002-2012, OpenNebula Project Leads (OpenNebula.org)             #
@@ -19,45 +19,42 @@
 source $(dirname $0)/kvmrc
 source $(dirname $0)/../../scripts_common.sh
 
-#------------------------------------------------------------------------------
-# Wait the VM to shutdown TIMEOUT (xPOLL_INTERVAL) seconds.
-# Set to ~10min
-#------------------------------------------------------------------------------
-POLL_INTERVAL=2
-TIMEOUT=300
-HALF_LOOP=$(($TIMEOUT/POLL_INTERVAL))
+count=0
 
 deploy_id=$1
 
-virsh --connect $LIBVIRT_URI shutdown $deploy_id
-
-exit_code=$?
-
-if [ "$exit_code" != "0" ]; then
-    error_message "Could not shutdown $deploy_id"
-    exit $exit_code
+if [ -z "$SHUTDOWN_TIMEOUT" ]; then
+    TIMEOUT=120
+else
+    TIMEOUT=$SHUTDOWN_TIMEOUT
 fi
 
-count=0
-while [ $(virsh --connect $LIBVIRT_URI --readonly dominfo $deploy_id > /dev/null 2>&1; echo $?) = "0" ]
-do 
-	sleep $POLL_INTERVAL
-	if [ "$count" -gt "$TIMEOUT" ]
-	then
-	    error_message "Timeout reached and VM $deploy_id is still alive"
-		echo "Timeout reached" >&2
-		exit 1
-	fi
+HALF_LOOP=$(($TIMEOUT/2))
 
+function monitor
+{
     # Issue another shutdown to cover occasional libvirt lack of attention
-	if [ "$count" -eq "$HALF_LOOP" ] 
-	then
-		virsh --connect $LIBVIRT_URI shutdown $deploy_id
+    if [ "$count" -eq "$HALF_LOOP" ]
+    then
+        virsh --connect $LIBVIRT_URI shutdown $deploy_id
     fi
 
-	let count=count+$POLL_INTERVAL
-done
+    let count=count+1
+
+    export count
+
+    virsh --connect $LIBVIRT_URI --readonly dominfo $deploy_id > /dev/null 2>&1
+
+    [ "x$?" != "x0" ]
+}
+
+exec_and_log "virsh --connect $LIBVIRT_URI shutdown \"$deploy_id\"" \
+    "Could not shutdown $deploy_id"
+
+retry $TIMEOUT monitor
+
+force_shutdown "$deploy_id" \
+    "virsh --connect $LIBVIRT_URI destroy \"$deploy_id\""
 
 sleep 4
 
-exit 0

src/vmm_mad/remotes/xen/shutdown

@@ -21,24 +21,23 @@ source $(dirname $0)/../../scripts_common.sh
 
 deploy_id=$1
 
-function gdm {
-    $XM_LIST | grep "$deploy_id "
+if [ -z "$SHUTDOWN_TIMEOUT" ]; then
+    TIMEOUT=120
+else
+    TIMEOUT=$SHUTDOWN_TIMEOUT
+fi
+
+function monitor
+{
+    $XM_LIST "$deploy_id" > /dev/null
+
+    [ "x$?" != "x0" ]
 }
 
 exec_and_log "$XM_SHUTDOWN $deploy_id" \
     "Could not shutdown $deploy_id"
 
-OUT=$(gdm)
+retry $TIMEOUT monitor
 
-while [ -n "$OUT" -a "$(echo $OUT | awk '{print $5}')" != "---s--" ]; do
-    sleep 1
-    OUT=$(gdm)
-done
-
-OUT=$(gdm)
-
-if [ -n "$OUT" ]; then
-    $XM_CANCEL "$deploy_id"
-fi
-sleep 2
+force_shutdown "$deploy_id" "$XM_CANCEL \"$deploy_id\""
 

src/vmm_mad/remotes/xen/xenrc

@@ -28,4 +28,10 @@ export XM_LIST="sudo $XM_PATH list"
 export XM_SHUTDOWN="sudo $XM_PATH shutdown"
 export XM_POLL="sudo /usr/sbin/xentop -bi2"
 
+# Senconds to wait after shutdown until timeout
+export SHUTDOWN_TIMEOUT=120
+
+# Uncoment this line to force VM cancellation after shutdown timeout
+#export FORCE_DESTROY=yes
+