B #6078: intoduce retry_if/retry_if_no_error (#2478)

(cherry picked from commit 41ed5fa477adabae7f51b96b1b42fa77383fdc8d)

src/mad/sh/scripts_common.sh

@@ -158,6 +158,7 @@ function exclusive
 }
 
 # Retries if command fails and STDERR matches
+# exit on error
 function retry_if
 {
     MATCH=$1
@@ -184,6 +185,38 @@ function retry_if
         break
     done
 
+    [ -n "$TERR" ] && echo $TERR >&2
+    [ -n "$TSTD" ] && echo $TSTD
+    exit $RC
+}
+
+# Retries if command fails and STDERR matches
+function retry_if_no_error
+{
+    MATCH=$1
+    TRIES=$2
+    SLEEP=$3
+    shift 3
+
+    unset TSTD TERR RC
+
+    while [ $TRIES -gt 0 ]; do
+        TRIES=$(( TRIES - 1 ))
+
+        eval "$( ("$@" ) \
+                2> >(TERR=$(cat); typeset -p TERR) \
+                 > >(TSTD=$(cat); typeset -p TSTD); RC=$?; typeset -p RC )"
+
+        [ $RC -eq 0 ] && break
+
+        if echo "$TERR" | grep -q "$MATCH"; then
+            sleep $SLEEP;
+            continue
+        fi
+
+        break
+    done
+
     [ -n "$TERR" ] && echo $TERR >&2
     [ -n "$TSTD" ] && echo $TSTD
     return $RC

src/tm_mad/ssh/cpds

@@ -109,7 +109,7 @@ if [ "${LCM_STATE}" = '26' -a "${SNAP_ID}" = '-1' ]; then
 $(type retry_if| grep -v 'is a function')
 
 touch ${SRC_TEMP_PATH}
-if ! retry_if "active block job" 3 5 virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PATH} --wait --finish; then
+if ! retry_if_no_error "active block job" 3 5 virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PATH} --wait --finish; then
     set -e -o pipefail
 
     if virsh -c ${LIBVIRT_URI} domfsfreeze ${DEPLOY_ID}; then

src/tm_mad/ssh/snap_create_live

@@ -72,7 +72,7 @@ $(type retry_if| grep -v 'is a function')
 mkdir -p "${SNAP_DIR}"
 touch $SNAP_PATH
 
-retry_if "active block job" 3 5  virsh -q -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} \
+retry_if_no_error "active block job" 3 5  virsh -q -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} \
     --path ${SRC_PATH} --dest $SNAP_PATH --wait --finish
 
 if [ -n "$REPLICA_HOST" ]; then

src/vmm_mad/remotes/kvm/migrate

@@ -122,7 +122,7 @@ if [ "x$CLEANUP_MEMORY_ON_START" = "xyes" ]; then
 fi
 
 if [ "$SHARED" = "YES" ]; then
-    retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \
+    retry_if_no_error "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \
         --live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system
 
     RC=$?

src/vmm_mad/remotes/kvm/snapshot_create

@@ -43,7 +43,7 @@ DATASTORE_PATH="${XPATH_ELEMENTS[i++]}"
 # -------- Create the snapshot and dump its metadata to xml file ------------
 
 SNAP_NAME="snap-${SNAP_ID}"
-data=`retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
+data=`retry_if_no_error "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
 
 if [ "$?" = "0" ]; then
     echo "$data" | awk '{print $3}'
@@ -52,7 +52,7 @@ if [ "$?" = "0" ]; then
         SNAP_XML_PATH="${DATASTORE_PATH}/${SNAP_NAME}.xml"
 
         # dump snapshot metadata xml to the VM location
-        retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
+        retry_if_no_error "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
     fi
 else
     error_message "Could not create snapshot $NAME for domain $DOMAIN."