From dd01710590f754aaee14eb32dbe1e8493fbfad78 Mon Sep 17 00:00:00 2001
From: Jan Orel <jorel@opennebula.systems>
Date: Tue, 7 Feb 2023 08:16:16 +0100
Subject: [PATCH] B #6078: intoduce retry_if/retry_if_no_error (#2478)

(cherry picked from commit 41ed5fa477adabae7f51b96b1b42fa77383fdc8d)
---
 src/mad/sh/scripts_common.sh            | 33 +++++++++++++++++++++++++
 src/tm_mad/ssh/cpds                     |  2 +-
 src/tm_mad/ssh/snap_create_live         |  2 +-
 src/vmm_mad/remotes/kvm/migrate         |  2 +-
 src/vmm_mad/remotes/kvm/snapshot_create |  4 +--
 5 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh
index 486434fad3..7a015500b9 100644
--- a/src/mad/sh/scripts_common.sh
+++ b/src/mad/sh/scripts_common.sh
@@ -158,6 +158,7 @@ function exclusive
 }
 
 # Retries if command fails and STDERR matches
+# exit on error
 function retry_if
 {
     MATCH=$1
@@ -184,6 +185,38 @@ function retry_if
         break
     done
 
+    [ -n "$TERR" ] && echo $TERR >&2
+    [ -n "$TSTD" ] && echo $TSTD
+    exit $RC
+}
+
+# Retries if command fails and STDERR matches
+function retry_if_no_error
+{
+    MATCH=$1
+    TRIES=$2
+    SLEEP=$3
+    shift 3
+
+    unset TSTD TERR RC
+
+    while [ $TRIES -gt 0 ]; do
+        TRIES=$(( TRIES - 1 ))
+
+        eval "$( ("$@" ) \
+                2> >(TERR=$(cat); typeset -p TERR) \
+                 > >(TSTD=$(cat); typeset -p TSTD); RC=$?; typeset -p RC )"
+
+        [ $RC -eq 0 ] && break
+
+        if echo "$TERR" | grep -q "$MATCH"; then
+            sleep $SLEEP;
+            continue
+        fi
+
+        break
+    done
+
     [ -n "$TERR" ] && echo $TERR >&2
     [ -n "$TSTD" ] && echo $TSTD
     return $RC
diff --git a/src/tm_mad/ssh/cpds b/src/tm_mad/ssh/cpds
index 347abd1973..edbf31f45d 100755
--- a/src/tm_mad/ssh/cpds
+++ b/src/tm_mad/ssh/cpds
@@ -109,7 +109,7 @@ if [ "${LCM_STATE}" = '26' -a "${SNAP_ID}" = '-1' ]; then
 $(type retry_if| grep -v 'is a function')
 
 touch ${SRC_TEMP_PATH}
-if ! retry_if "active block job" 3 5 virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PATH} --wait --finish; then
+if ! retry_if_no_error "active block job" 3 5 virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PATH} --wait --finish; then
     set -e -o pipefail
 
     if virsh -c ${LIBVIRT_URI} domfsfreeze ${DEPLOY_ID}; then
diff --git a/src/tm_mad/ssh/snap_create_live b/src/tm_mad/ssh/snap_create_live
index 7a97e9487a..a5365d6c2a 100755
--- a/src/tm_mad/ssh/snap_create_live
+++ b/src/tm_mad/ssh/snap_create_live
@@ -72,7 +72,7 @@ $(type retry_if| grep -v 'is a function')
 mkdir -p "${SNAP_DIR}"
 touch $SNAP_PATH
 
-retry_if "active block job" 3 5  virsh -q -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} \
+retry_if_no_error "active block job" 3 5  virsh -q -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} \
     --path ${SRC_PATH} --dest $SNAP_PATH --wait --finish
 
 if [ -n "$REPLICA_HOST" ]; then
diff --git a/src/vmm_mad/remotes/kvm/migrate b/src/vmm_mad/remotes/kvm/migrate
index 7b0ad413a8..0874c0e2d4 100755
--- a/src/vmm_mad/remotes/kvm/migrate
+++ b/src/vmm_mad/remotes/kvm/migrate
@@ -122,7 +122,7 @@ if [ "x$CLEANUP_MEMORY_ON_START" = "xyes" ]; then
 fi
 
 if [ "$SHARED" = "YES" ]; then
-    retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \
+    retry_if_no_error "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \
         --live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system
 
     RC=$?
diff --git a/src/vmm_mad/remotes/kvm/snapshot_create b/src/vmm_mad/remotes/kvm/snapshot_create
index 1b7342a7f6..16d49100aa 100755
--- a/src/vmm_mad/remotes/kvm/snapshot_create
+++ b/src/vmm_mad/remotes/kvm/snapshot_create
@@ -43,7 +43,7 @@ DATASTORE_PATH="${XPATH_ELEMENTS[i++]}"
 # -------- Create the snapshot and dump its metadata to xml file ------------
 
 SNAP_NAME="snap-${SNAP_ID}"
-data=`retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
+data=`retry_if_no_error "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
 
 if [ "$?" = "0" ]; then
     echo "$data" | awk '{print $3}'
@@ -52,7 +52,7 @@ if [ "$?" = "0" ]; then
         SNAP_XML_PATH="${DATASTORE_PATH}/${SNAP_NAME}.xml"
 
         # dump snapshot metadata xml to the VM location
-        retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
+        retry_if_no_error "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
     fi
 else
     error_message "Could not create snapshot $NAME for domain $DOMAIN."